Reorganize python word impls, switch tokenizer to a stream approach.

sallyforth/basic_words.py

@@ -0,0 +1,280 @@
+from inspect import isfunction, isbuiltin
+import importlib
+import os
+from compiler import Compiler
+from arglist import Arglist
+
+def const_f(value):
+    def x(f, i):
+        #print("const f, pushing", value)
+        f.stack.push(value)
+        return i + 1
+    return x
+
+def native_function_handler(func):
+    def handle(forth, i):
+        args = forth.stack.pop()
+        #print(f"Native fun, calling {func}({args})")
+        result = func(*args)
+        #print(f'Result: {result}')
+        forth.stack.push(result)
+        #print("pushed result")
+        return i + 1
+    return handle
+
+def import_native_module(forth, m, alias=None, excludes=[]):
+    if not alias:
+        alias = m.__name__
+    raw_names = dir(m)
+    names = [x for x in raw_names if x not in excludes] 
+    for name in names:
+        localname = f'{alias}.{name}'
+        val = m.__getattribute__(name)
+        forth.namespace[localname] = const_f(val)
+
+def w_nexttoken(f, i):
+    token = f.read_next_token()
+    f.stack.push(token)
+    return i+1
+
+def w_eval(f, i):
+    token = f.stack.pop()
+    f.execute_token(token)
+    return i+1
+
+def w_no_op(f, i):
+    return i+1
+
+def w_forth(f, i):
+    f.stack.push(f)
+    return i+1
+
+def w_current_ns(f, i):
+    f.stack.push(f.namespace)
+    return i + 1
+
+def w_ns(f, i):
+    name = f.stack.pop()
+    if name in f.namespaces:
+        f.namespace = f.namespaces[name]
+    else:
+        new_ns = f.make_namespace(name, {}, [f.forth_ns])
+        f.namespace = new_ns
+    return i + 1
+
+def w_alias(f, i):
+    new_name = f.stack.pop() 
+    old_name = f.stack.pop() 
+    f.namespace[new_name] = f.namespace[old_name]
+    return i + 1
+
+def w_require(f, i):
+    name = f.stack.pop()
+    m = importlib.import_module(name)
+    import_native_module(f, m, name)
+    return i + 1
+
+def source(f, path):
+    old_source_f = f.namespace.get('*source*', None)
+    old_namespace = f.namespace
+    try:
+        f.execute_file(path)
+    finally:
+        f.namespace['*source*'] = old_source_f
+        f.namespace = old_namespace
+
+def w_load(f, i):
+    path = f.stack.pop()
+    source(f, path)
+    return i + 1
+
+def w_source(f, i):
+    path = f.stack.pop()
+    if os.path.isabs(path):
+        source(f, path)
+        return i+1
+
+    relative_dir = os.path.dirname(f.evaluate_string('*source*'))
+    relative_path = f'{relative_dir}/{path}'
+    if os.path.exists(relative_path):
+        source(f, relative_path)
+        return i+1
+
+    source(f, path)
+    return i+1
+
+
+def execute_f(name, instructions):
+    #print('execute_f:', name, len(instructions))
+    def inner(forth, i, debug=False):
+        #print('inner f:', name)
+        #print('inner f:', len(instructions))
+        j = 0
+        while j >= 0:
+            #print(j, '=>', instructions[j])
+            new_j = instructions[j](forth, j)
+            if new_j == None:
+                print(f'Instruction {instructions[j]} None')
+                raise RuntimeError
+            j = new_j
+        return i + 1
+    return inner
+
+def ifnot_jump_f(n):
+    def ifnot_jump(forth, i):
+        x = forth.stack.pop()
+        if not x:
+            return i+n
+        return i+1
+    return ifnot_jump
+
+def jump_f(n):
+    def do_jump(forth, i):
+        return n+i
+    return do_jump
+
+def w_recur(f, i):
+    return 0
+
+def w_import(f, i):
+    name = f.stack.pop()
+    m = importlib.import_module(name)
+    f.namespace[name] = const_f(m)
+    return i+1
+
+def w_call(f, i):
+    func = f.stack.pop()
+    args = f.stack.pop()
+    # print('f', f, 'args', args)
+    try:
+        result = func(*args)
+    except:
+        print(f'Error executing {func}{list(args)}')
+        raise
+    # print('result', result)
+    f.stack.push(result)
+    return i+1
+
+def w_nl(f, i):
+    print()
+    return i+1
+
+def w_return(f, i):
+    return -9999;
+
+def w_colon(f, i):
+    f.compiler = Compiler()
+
+def w_semi(forth, i):
+    forth.compiler.add_instruction(w_return)
+    name = forth.compiler.name
+    word_f = execute_f(name, forth.compiler.instructions)
+    forth.namespace[name] = word_f
+    forth.compiler = None
+    return i+1
+w_semi.__dict__['immediate'] = True
+
+def w_should_not_happen(forth, i):
+    print('Should not execute this word!')
+    raise ValueError
+
+def w_if(forth, i):
+    #print('w_if')
+    compiler = forth.compiler
+    compiler.push_offset()
+    compiler.push_offset()
+    compiler.add_instruction(w_should_not_happen)
+    return i+1
+
+w_if.__dict__['immediate'] = True
+
+def w_then(forth, i):
+    compiler = forth.compiler
+    else_offset = compiler.pop_offset()
+    if_offset = compiler.pop_offset()
+    then_offset = compiler.offset()
+    if else_offset == if_offset:
+        delta = then_offset - if_offset
+        compiler.instructions[if_offset] = ifnot_jump_f(delta)
+    else:
+        if_delta = else_offset - if_offset + 1
+        compiler.instructions[if_offset] = ifnot_jump_f(if_delta)
+        else_delta = then_offset - else_offset
+        compiler.instructions[else_offset] = jump_f(else_delta)
+    return i+1
+
+w_then.__dict__['immediate'] = True
+
+
+def w_else(forth, i):
+    compiler = forth.compiler
+    compiler.pop_offset()
+    compiler.push_offset()
+    compiler.add_instruction(w_should_not_happen)
+    return i+1
+
+w_else.__dict__['immediate'] = True
+
+def w_do(forth, i):
+    #print('w_do')
+    compiler = forth.compiler
+    compiler.push_offset()
+    compiler.add_instruction(w_should_not_happen)
+    return i+1
+
+w_do.__dict__['immediate'] = True
+
+def w_while(forth, i):
+    compiler = forth.compiler
+    do_offset = compiler.pop_offset()
+    while_offset = compiler.offset()
+    delta = do_offset - while_offset
+    compiler.instructions[if_offset] = ifnot_jump_f(delta)
+    return i+1
+
+w_while.__dict__['immediate'] = True
+
+def w_begin(forth, i):
+    compiler = forth.compiler
+    compiler.push_offset()
+    return i+1
+
+w_begin.__dict__['immediate'] = True
+
+def w_until(forth, i):
+    compiler = forth.compiler
+    begin_offset = compiler.pop_offset()
+    until_offset = compiler.offset()
+    delta = begin_offset - until_offset
+    #print('Delta:', delta)
+    compiler.instructions.append(ifnot_jump_f(delta))
+    return i+1
+
+
+w_until.__dict__['immediate'] = True
+
+def w_dump(f, i):
+    f.dump()
+    return i+1
+
+def w_idump(f, i):
+    f.dump()
+    return i+1
+
+w_idump.__dict__['immediate'] = True
+
+def w_stack(f, i):
+    print("Stack:", end=' ')
+    for x in f.stack:
+        print(f'{repr(x)}', end=' ')
+    print()
+    return i+1
+    
+def w_enlist(f, i):
+    # print("Enlist!")
+    x = f.stack.pop()
+    # print("Popped", x)
+    f.stack.push([x])
+    return i+1
+
+

sallyforth/data_words.py

@@ -0,0 +1,157 @@
+from unique import Unique
+
+def w_unique(f, ip):  # pushes a uique object.
+    f.stack.push(Unique())
+    return ip+1
+
+def w_map(f, ip):
+    word = f.stack.pop()
+    l = f.stack.pop()
+
+    word_f = f.namespace.get(word, None)
+    result = []
+
+    for item in l:
+        f.stack.push(item)
+        word_f(f, 0)
+        result.append(f.stack.pop())
+
+    f.stack.push(result)
+    return ip+1
+
+def w_reduce(f, ip):
+    l = f.stack.pop()
+    word = f.stack.pop()
+
+    word_f = f.namespace.get(word, None)
+
+    if len(l) <= 0:
+        f.stack.push(None)
+    elif len(l) == 1:
+        f.stack.push(l[0])
+    else:
+        result = l[0]
+        l = l[1::-1]
+        for item in l:
+            f.stack.push(result)
+            f.stack.push(item)
+            word_f(f, 0)
+            result = f.stack.pop()
+        f.stack.push(result)
+
+    return ip+1
+
+def w_bounded_list(f, ip):
+    """Create a list from delimted values on the stack.
+    [list]
+    (marker a b c marker -- [a b c]
+    """
+    marker = f.stack.pop()
+    l = []
+    if f.stack.empty():
+        raise ValueError("Stack underflow")
+    x = f.stack.pop()
+    while x != marker:
+        l.append(x)
+        if f.stack.empty():
+            raise ValueError("Stack underflow")
+        x = f.stack.pop()
+    l.reverse()
+    f.stack.push(l)
+    return ip+1
+
+def w_to_arglist(f, ip):
+    l = f.stack.pop()
+    f.stack.push(Arglist(l))
+    return ip+1
+
+def w_list(f, ip):    # ->list
+    n = f.stack.pop()
+    l = []
+    for i in range(n):
+        l.append(f.stack.pop())
+    f.stack.push(l)
+    return ip+1
+
+def w_thread(f, i):    # @@
+    contents = f.stack.pop()
+    result = contents[0]
+    for field in contents[1::]:
+        if isinstance(field, str) and hasattr(result, field):
+            result = getattr(result, field)    # result.field
+        elif isinstance(field, Arglist):
+            result = result(*field)            # result(*field)
+        else:
+            result = result[field]             # result[field]
+    f.stack.push(result)
+    return i+1
+
+
+ListMarker = object()
+
+def w_startlist(f, i):   # [
+    f.stack.push(ListMarker)
+    return i+1
+
+def w_endlist(f, i):    # ]
+    l = []
+    x = f.stack.pop()
+    while x != ListMarker:
+        l.append(x)
+        x = f.stack.pop()
+    l.reverse()
+    f.stack.push(l)
+    return i+1
+
+MapMarker = object()
+
+def w_startmap(f, i):   # {
+    f.stack.push(MapMarker)
+    return i+1
+
+def w_endmap(f, ip):    # }
+    l = []
+    x = f.stack.pop()
+    while x != MapMarker:
+        l.append(x)
+        x = f.stack.pop()
+    if (len(l) % 2) != 0:
+        print('Maps need even number of entries.')
+        return i+1
+    l.reverse()
+    result = {}
+    for i in range(0, len(l), 2):
+        result[l[i]] = l[i+1]
+    f.stack.push(result)
+    return ip+1
+
+def w_list_to_map(f, ip):  # list->map
+    l = f.stack.pop()
+    result = {}
+    for i in range(0, len(l), 2):
+        result[l[i]] = l[i+1]
+    f.stack.push(result)
+    return ip+1
+
+def w_get(f, i):
+    name = f.stack.pop()
+    m = f.stack.pop()
+    result = m[name]
+    f.stack.push(result)
+    return i+1
+
+def w_getattribute(f, i):
+    name = f.stack.pop()
+    x = f.stack.pop()
+    result = x.__getattribute__(name)
+    f.stack.push(result)
+    return i+1
+
+def w_def(f, i):
+    value = f.stack.pop()
+    name = f.stack.pop()
+    f.defvar(name, value)
+    # print('name', name, 'value', value)
+    return i+1
+
+

sallyforth/init.sf

@@ -0,0 +1,47 @@
+'*prompt* "sallySh> " def
+
+: child-run-prog (argv pid -- <<exit>>)
+  drop            \ Get rid of 0 pid.
+  execvp          \ Run the program
+;
+
+: parent-wait (cmd pid -- exit-status)
+  "parent" p
+  waitpid
+  "Child status:" p p
+  drop
+;
+
+: run-prog (argv -- status)
+  'child-run-prog 'parent-wait fork
+;
+
+'cmd-marker unique def
+
+: a cmd-marker ;
+
+: z 
+  cmd-marker bounded_list
+  stack
+  run-prog
+;
+
+'os import
+
+: expandvars #os.path.expandvars !!1 ;
+: expanduser #os.path.expanduser !!1 ;
+: expand expanduser expandvars ;
+
+\: prompt-and-run ( -- prog-status)
+\  ">> " read-line
+\  dup "x" = 
+\  if
+\    "Exit!" p
+\    drop
+\  else
+\    tokenize
+\   'expand map
+\    run-prog
+\    recur
+\  then
+\;

sallyforth/io.sf

@@ -0,0 +1,12 @@
+
+: open builtins.open  !!1 ;
+: close <. $? 'close .> !!0 drop  ;
+
+: read-file (path -- contents) open dup <. $? 'read .> !!0 swap close ;
+: read-lines (path -- contents) open dup <. $? 'readlines .> !!0 swap close ;
+
+: read-line (prompt -- input-line)
+  builtins.input !!1
+;
+
+: read-next nexttoken second ;

sallyforth/kernel.py

@@ -1,8 +1,9 @@
 import sys
 from os import path
-from words import *
+import basic_words, data_words, operator_words, stack_words, os_words
-import words
+from basic_words import const_f, w_enlist
-from lex import  is_string, Tokenizer
+#from lex import  is_string, Tokenizer
+import tokenstream as ts
 from stack import Stack
 from namespace import Namespace
 
@@ -17,7 +18,7 @@ def to_number(token):
 
 class Forth:
     def __init__(self, startup=None):
-        self.tokenizer = Tokenizer(self)
+        self.streams = Stack()
         self.stack = Stack()
         self.namespaces = {}
         initial_defs = {
@@ -34,7 +35,11 @@ class Forth:
         self.forth_ns = self.make_namespace('forth', initial_defs)
         user_ns = self.make_namespace('user', {}, [self.forth_ns])
 
-        self.forth_ns.import_from_module(words, 'w_')
+        self.forth_ns.import_from_module(basic_words, 'w_')
+        self.forth_ns.import_from_module(data_words, 'w_')
+        self.forth_ns.import_from_module(operator_words, 'w_')
+        self.forth_ns.import_from_module(stack_words, 'w_')
+        self.forth_ns.import_from_module(os_words, 'w_')
         self.namespace = self.forth_ns
 
         self.compiler = None
@@ -49,55 +54,120 @@ class Forth:
     def defvar(self, name, value):
         self.namespace[name] = const_f(value)
 
-    def py_evaluate(self, token, *args):
+    def compiling(self):
-        #print(f'Evaluate: token [{token}] args <<{args}>>')
+        return self.compiler
+
+    def _compile_token(self, kind, token):
+        print(f"compile: {self.compiler.name}: {kind} {token}")
+        if self.compiler.name == None:
+            self.compiler.name = token
+            return
+
+        if kind in ['dqstring', 'sqstring']:
+            self.compiler.add_instruction(const_f(token))
+            return
+
+        if token in self.namespace:
+            word = self.namespace[token]
+            if 'immediate' in word.__dict__:
+                word(self, 0)
+            else:
+                self.compiler.add_instruction(self.namespace[token])
+            return
+
+        n = to_number(token)
+        if n == None:
+            print(f'{token}? Compile of {self.compiler.name} terminated.')
+            self.compiler = None
+        else:
+            self.compiler.add_instruction(const_f(n))
+
+    def _eval_token(self, kind, token):
+        if kind in ['dqstring', 'sqstring']:
+            self.stack.push(token)
+            return
+
+        if token in self.namespace:
+            # print("executing ", token)
+            self.namespace[token](self, 0)
+            return
+
+        n = to_number(token)
+        if n == None:
+            print(f'{token}?')
+        else:
+            self.stack.push(n)
+
+    def execute_token(self, kind, token):
+        #print(f'execute kind {kind} token: {token}')
+        kts = self.macro_expand_token(kind, token)
+        #print(kts)
+        for kt in kts:
+            this_kind, this_token = kt
+            #print(f'execute this {this_kind} {this_token}')
+            if not self.compiling():
+                self._eval_token(this_kind, this_token)
+            else:
+                self._compile_token(this_kind, this_token)
+
+    def XX_execute_token_stream(self, s):
+        kind, token = s.get_token()
+        while kind != 'eof':
+            self.execute_token(kind, token)
+            kind, token = s.get_token()
+
+    def execute_current_stream(self):
+        s = self.streams.peek()
+        print("exec current s:", s)
+        kind, token = s.get_token()
+        while kind != 'eof':
+            self.execute_token(kind, token)
+            kind, token = s.get_token()
+        self.streams.pop()
+
+    def execute_token_stream(self, s):
+        print("exec token stream:", s)
+        self.streams.push(s)
+        self.execute_current_stream()
+
+    def execute_string(self, s):
+        token_stream = ts.string_token_stream(s)
+        return self.execute_token_stream(token_stream)
+
+    def evaluate_string(self, s):
+        self.execute_string(s)
+        return self.stack.pop()
+
+    def read_next_token(self):
+        s = self.streams.peek()
+        return s.get_token()
+
+    def py_evaluate(self, s, *args):
+        print(f'Evaluate: token [{token}] args <<{args}>>')
         rargs = list(args)
         rargs.reverse()
         if rargs:
             for a in rargs:
                 # print("pushing", a);
                 self.stack.push(a)
-        #print(f'Before eval stack is {str(self.stack)}')
+        print(f'Before eval stack is {str(self.stack)}')
-        return self.evaluate_token(token)
+        return self.evaluate_string(s)
 
-    def evaluate_token(self, token):
-        #print("evaluate token: ", token)
-        self.execute_token(token)
-        return self.stack.pop()
 
-    def compiling(self):
+    def macro_expand_token(self, kind, token):
-        return self.compiler
-
-    def execute_line(self, line):
-        tokens = self.tokenizer.tokenize(line)
-        self.execute_tokens(tokens)
-
-    def execute_tokens(self, tokens):
-        for token in tokens:
-            # print("token:", token)
-            if not self.compiling():
-                self.execute_token(token)
-            else:
-                self.compile_token(token)
-
-    def macro_expand_token(self, token):
         if len(token) <= 0 or token[0] != '#':
-            return [token]
+            return [[kind, token]]
 
         tag = token[1:]
         parts = tag.split('.')
         result = [ '<.', parts[0] ]
+        result = [['word', '<.'], ['word', parts[0]]]
         for part in parts[1::]:
-            result.append("'" + part)
+            result.append(['sqstring', part])
-        result.append('.>')
+        result.append(['word', '.>'])
+        print(result)
         return result
 
-    def macro_expand_tokens(self, tokens):
-        results = []
-        for token in tokens:
-            results.extend(self.macro_expand_token(token))
-        return results
-
     def set_ns(self, ns_name):
         if ns_name in self.namespaces:
             self.namespace = self.namespaces[ns_name]
@@ -116,54 +186,11 @@ class Forth:
         old_namespace = self.namespace
         self.defvar('*source*', fpath)
         with open(fpath) as f:
-            line = f.readline()
+            fts = ts.file_token_stream(f)
-            while line:
+            self.execute_token_stream(fts)
-                self.execute_line(line)
-                line = f.readline()
         self.namespace['*source*'] = old_source
         self.namespace = old_namespace
 
-    def compile_token(self, token):
-        if self.compiler.name == None:
-            self.compiler.name = token
-            return
-
-        if is_string(token):
-            self.compiler.add_instruction(const_f(token[1::]))
-            return
-
-        if token in self.namespace:
-            word = self.namespace[token]
-            if 'immediate' in word.__dict__:
-                word(self, 0)
-            else:
-                self.compiler.add_instruction(self.namespace[token])
-            return
-
-        n = to_number(token)
-        if n == None:
-            print(f'{token}? Compile of {self.compiler.name} terminated.')
-            self.compiler = None
-        else:
-            self.compiler.add_instruction(const_f(n))
-
-    def execute_token(self, token):
-        # print("x token:", token)
-        if is_string(token):
-            self.stack.push(token[1::])
-            return
-
-        if token in self.namespace:
-            # print("executing ", token)
-            self.namespace[token](self, 0)
-            return
-
-        n = to_number(token)
-        if n == None:
-            print(f'{token}?')
-        else:
-            self.stack.push(n)
-
     def dump(self):
         print('Forth:', self)
         print('Stack:', self.stack)

sallyforth/lex.py

@@ -1,7 +1,3 @@
-import sys
-import readline
-from os import path
-
 def is_string(token):
     return token[0] == '"' or token[0] == "'"
 

sallyforth/list.sf

@@ -20,7 +20,7 @@
   [x]                               \ Do a[0..n].
 ;
  
-: drop (n list -- all-but-first-n-items)
+: skip (n list -- all-but-first-n-items)
   swap nil slice                    \ Make the n..None slice.
   [x]
 ;
@@ -29,7 +29,11 @@
   1 ->list * 
 ;
 
-: rest   (list -- all-but-first) 1 swap drop ;
+: len builtins.len !!1 ;
+
+: empty? len zero? ;
+  
+: rest   (list -- all-but-first) 1 swap skip ;
 : rrest  (list -- rest-of-rest) rest rest ;
 : rrrest (list -- all-but-first) rest rest rest ; 
 

sallyforth/namespace.py

@@ -38,7 +38,9 @@ class Namespace:
         return self[key]
         
     def __contains__(self, key):
+        #print(f'Namespace contains {key}')
         if self.contents.__contains__(key):
+            #print(self.contents[key])
             return True
         for r in self.refers:
             if r.__contains__(key):

sallyforth/operator_words.py

@@ -0,0 +1,67 @@
+def w_gt(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b > a)
+    return i+1
+
+def w_lt(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b < a)
+    return i+1
+
+def w_eq(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(a==b)
+    return i+1
+
+def w_le(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b<=a)
+    return i+1
+
+def w_ge(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b>=a)
+    return i+1
+
+def w_add(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b+a)
+    return i+1
+
+def w_mul(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b*a)
+    return i+1
+
+def w_sub(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b-a)
+    return i+1
+
+def w_div(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b/a)
+    return i+1
+
+def w_and(f, i):
+    f.stack.push(f.stack.pop() and f.stack.pop())
+    return i+1
+
+def w_or(f, i):
+    f.stack.push(f.stack.pop() or f.stack.pop())
+    return i+1
+
+def w_not(f, i):
+    f.stack.push(not f.stack.pop())
+    return i+1
+
+

sallyforth/os_words.py

@@ -0,0 +1,41 @@
+import os
+import sys
+
+def w_fork(f, i):
+    parent_word = f.stack.pop()
+    child_word = f.stack.pop()
+    parent_f = f.namespace.get(parent_word, None)
+    child_f = f.namespace.get(child_word, None)
+    pid = os.fork()
+    f.stack.push(pid)
+    if pid == 0:
+        print("child:", pid)
+        child_f(f, 0)
+    else:
+        print("parent:", pid)
+        parent_f(f, 0)
+    return i+1
+
+def w_execvp(f, i):
+    args = f.stack.pop()
+    path = args[0]
+    print(f"path {path} args: {args}")
+    os.execvp(path, args)
+    return i+1
+
+def w_waitpid(f, i):
+    pid = f.stack.pop()
+    result = os.waitpid(pid, 0)
+    f.stack.push(result)
+    return i+1
+
+def w_exit(f, i):
+    n = f.stack.pop()
+    sys.exit(n)
+    return i+1
+
+def w_exit_bang(f, i):
+    n = f.stack.pop()
+    os._exit(n)
+    return i+1
+

sallyforth/sallyforth.py

@@ -1,9 +1,10 @@
 import os
 import sys
 import atexit
-from kernel import Forth
 import readline
 import traceback
+from kernel import Forth
+from tokenstream import prompt_token_stream
 
 HistoryFile=".sallyforth"
 
@@ -30,6 +31,9 @@ def setup_readline(history_path, f):
     def save_history():
         readline.write_history_file(history_path)
     atexit.register(save_history)
+    def prompt_f():
+        return f.evaluate_string('*prompt*')
+    return prompt_token_stream(prompt_f)    
 
 def setup_forth():
     source_dir = os.path.dirname(os.path.abspath(__file__))
@@ -39,34 +43,13 @@ def setup_forth():
         f = Forth(startup_file)
     else:
         f = Forth()
-
-
     return f
 
-def repl(f):
+def repl(stream, f):
-    while True:
+    f.execute_token_stream(stream)
-        p = f.evaluate_token('*prompt*')
-        try:
-            line = input(p)
-        except KeyboardInterrupt:
-            print("<<interrupt>>")
-            f.stack.reset()
-            line = ''
-        except EOFError:
-            break
-    
-        try:
-            f.execute_line(line)
-        except:
-            exc_type, exc_value, exc_traceback = sys.exc_info()
-            print("Error:", exc_type)
-            print("Error:", exc_value)
-            print("Error:", exc_traceback)
-            traceback.print_tb(exc_traceback)
-    
 
 if __name__ == "__main__":
     f = setup_forth()
-    setup_readline(hist_file, f)
+    stream = setup_readline(hist_file, f)
-    repl(f)
+    repl(stream, f)
     print("Bye!")

sallyforth/stack.py

@@ -4,14 +4,14 @@ class Stack:
         self.stack = 100 * [None]
 
     def push(self, x):
-        # print("stack push", x)
+        #print("stack push", x)
         self.top += 1
         self.stack[self.top] = x
         return x
 
     def pop(self):
         result = self.stack[self.top]
-        # print("stack pop", result)
+        #print("stack pop", result)
         self.top -= 1
         if self.top < -1:
             print("stack overpop")

sallyforth/stack_words.py

@@ -0,0 +1,103 @@
+def w_px(f, i):
+    args = f.stack.pop()
+    name = f.stack.pop()
+    m = f.stack.pop()
+    func = m.__dict__[name]
+    result = func(*args)
+    f.stack.push(result)
+    return i+1
+
+def w_reset(f, i):
+    a = f.stack.reset()
+    return i+1
+
+def w_dot(f, i):
+    a = f.stack.pop()
+    print(a, end='')
+    return i+1
+
+def w_splat(f, i):
+    l = f.stack.pop()
+    l.reverse()
+    for x in l:
+        f.stack.push(x)
+    return i+1
+
+def w_dup(f, i):
+    x = f.stack.peek()
+    f.stack.push(x)
+    return i+1
+
+def w_tmb(f, i):  # A noop
+    # t = f.stack.pop()
+    # m = f.stack.pop()
+    # b = f.stack.pop()
+    # f.stack.push(b)
+    # f.stack.push(m)
+    # f.stack.push(t)
+    return i+1
+
+def w_tbm(f, i):
+    t = f.stack.pop()
+    m = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(m)
+    f.stack.push(b)
+    f.stack.push(t)
+    return i+1
+
+def w_bmt(f, i):
+    t = f.stack.pop()
+    m = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(t)
+    f.stack.push(m)
+    f.stack.push(b)
+    return i+1
+
+def w_btm(f, i):
+    t = f.stack.pop()
+    m = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(m)
+    f.stack.push(t)
+    f.stack.push(b)
+    return i+1
+
+def w_mtb(f, i):
+    t = f.stack.pop()
+    m = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(b)
+    f.stack.push(t)
+    f.stack.push(m)
+    return i+1
+
+def w_mbt(f, i):
+    t = f.stack.pop()
+    m = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(t)
+    f.stack.push(b)
+    f.stack.push(m)
+    return i+1
+
+def w_rot(f, i):
+    c = f.stack.pop()
+    b = f.stack.pop()
+    a = f.stack.pop()
+    f.stack.push(b)
+    f.stack.push(c)
+    f.stack.push(a)
+    return i+1
+
+def w_drop(f, i):
+    f.stack.pop()
+    return i+1
+
+def w_swap(f, i):
+    a = f.stack.pop()
+    b = f.stack.pop()
+    f.stack.push(a)
+    f.stack.push(b)
+    return i+1

sallyforth/startup.sf

@@ -18,6 +18,7 @@
 'colon        ':          alias
 'semi         ';          alias
 'thread       '@@         alias
+'exit_bang    'exit!      alias
 'bounded_list '[list]     alias
 'list         '->list     alias
 'to_arglist   '->arglist  alias
@@ -47,6 +48,7 @@
 \ Look up attributes on a value.
 : <. [ ;
 : .> ] thread ;
+: $? swap ;
 
 \ Call native functions with various # arguments.
 : !!0 [] swap !! ;
@@ -60,6 +62,7 @@
 
 : {} ( -- <empty map>) { } ;
 
+
 \ Make a set.
 
 'set-marker unique def
@@ -87,6 +90,8 @@
 : hello "Hello" . nl ;
 
 : >0 0 > ;
+: =0 0 = ;
+: <1 1 < ;
 : <0 0 < ;
 : >1 1 > ;
 : <1 1 < ;
@@ -118,7 +123,10 @@
 : .!!2 (obj a1 a2 method-name -- result ) swap 2 ->list swap .!! ;
 : .!!3 (obj a1 a2 a3 method-name -- result ) swap 3 ->list swap .!! ;
 
+\ todo : tokenize #forth.tokenizer.tokenize  !!1 ;
+
 "string.sf" source
 "list.sf" source
+"io.sf" source
 
 "init.sf" source-if-exists

sallyforth/tokenstream.py

@@ -0,0 +1,87 @@
+import io
+
+class PromptInputStream:
+    def __init__(self, prompt_f):
+        self.prompt_f = prompt_f
+        self.buffer = []
+
+    def getc(self):
+        try:
+            if len(self.buffer) == 0:
+                prompt = self.prompt_f()
+                line = input(prompt)
+                line += '\n'
+                self.buffer = list(line)
+                self.buffer.reverse()
+            return self.buffer.pop()
+        except EOFError:
+            return ''
+
+class TokenStream:
+    def __init__(self, read_f):
+        self.read_f = read_f
+
+    def whitespace(self, ch):
+        return ch in [' ', '\t', '\n']
+
+    def get_token(self):
+        state = 'start'
+        token = ''
+        while True:
+            ch = self.read_f()
+            #print(f'ch: {ch} typech {type(ch)} state {state}')
+            if ch in ['', None]:
+                if state in ['word', 'sqstring', 'dqstring']:
+                    return [state, token]
+                return ['eof', '']
+            elif state == 'start' and ch == '\\':
+                state = 'lcomment'
+            elif state == 'lcomment' and ch == '\n':
+                state = 'start'
+            elif state == 'start' and ch == '(':
+                state = 'icomment'
+            elif state == 'icomment' and ch == ')':
+                state = 'start'
+            elif state == 'start' and self.whitespace(ch):
+                continue
+            elif state == 'start' and ch == '"':
+                state = 'dqstring'
+            elif state == 'dqstring' and ch == '"':
+                return [state, token]
+            elif state == 'start' and ch == "'":
+                state = 'sqstring'
+            elif state == 'start':
+                state = 'word'
+                token += ch
+            elif state in ['word', 'sqstring'] and \
+                 self.whitespace(ch):
+                return state, token
+            elif state in ['word', 'dqstring', 'sqstring']:
+                token += ch
+
+def file_token_stream(f):
+    return TokenStream(lambda : f.read(1))
+
+def string_token_stream(s):
+    sio = io.StringIO(s)
+    return file_token_stream(sio)
+
+def prompt_token_stream(prompt_f):
+    pis = PromptInputStream(prompt_f)
+    return TokenStream(pis.getc)
+
+if __name__ == "__main__":
+    x = 0
+    
+    def pmt():
+        global x
+        x += 1
+        return f'Yes{x}>> '
+    
+    pis = PromptInputStream(pmt)
+    ts = TokenStream(pis.getc)
+    
+    kind, token = ts.get_token()
+    while kind != 'eof':
+        print(kind, token)
+        kind, token = ts.get_token()

sallyforth/unique.py

@@ -0,0 +1,3 @@
+class Unique:
+    def __str__(self):
+        return f'Unique[{id(self)}]'

sallyforth/words.py

@@ -1,583 +0,0 @@
-from inspect import isfunction, isbuiltin
-import importlib
-import os
-from compiler import Compiler
-from arglist import Arglist
-
-class Unique:
-    def __str__(self):
-        return f'Unique[{id(self)}]'
-
-def const_f(value):
-    def x(f, i):
-        f.stack.push(value)
-        return i + 1
-    return x
-
-def native_function_handler(func):
-    def handle(forth, i):
-        args = forth.stack.pop()
-        #print(f"Native fun, calling {func}({args})")
-        result = func(*args)
-        #print(f'Result: {result}')
-        forth.stack.push(result)
-        #print("pushed result")
-        return i + 1
-    return handle
-
-def import_native_module(forth, m, alias=None, excludes=[]):
-    if not alias:
-        alias = m.__name__
-    raw_names = dir(m)
-    names = [x for x in raw_names if x not in excludes] 
-    for name in names:
-        localname = f'{alias}.{name}'
-        val = m.__getattribute__(name)
-        forth.namespace[localname] = const_f(val)
-
-def w_eval(f, i):
-    token = f.stack.pop()
-    f.execute_token(token)
-    return i+1
-
-def w_no_op(f, i):
-    return i+1
-
-def w_enlist(f, i):
-    # print("Enlist!")
-    x = f.stack.pop()
-    # print("Popped", x)
-    f.stack.push([x])
-    return i+1
-
-def w_forth(f, i):
-    f.stack.push(f)
-    return i+1
-
-def w_current_ns(f, i):
-    f.stack.push(f.namespace)
-    return i + 1
-
-def w_ns(f, i):
-    name = f.stack.pop()
-    if name in f.namespaces:
-        f.namespace = f.namespaces[name]
-    else:
-        new_ns = f.make_namespace(name, {}, [f.forth_ns])
-        f.namespace = new_ns
-    return i + 1
-
-def w_alias(f, i):
-    new_name = f.stack.pop() 
-    old_name = f.stack.pop() 
-    f.namespace[new_name] = f.namespace[old_name]
-    return i + 1
-
-def w_require(f, i):
-    name = f.stack.pop()
-    m = importlib.import_module(name)
-    import_native_module(f, m, name)
-    return i + 1
-
-def source(f, path):
-    old_source_f = f.namespace.get('*source*', None)
-    old_namespace = f.namespace
-    try:
-        f.execute_file(path)
-    finally:
-        f.namespace['*source*'] = old_source_f
-        f.namespace = old_namespace
-
-def w_load(f, i):
-    path = f.stack.pop()
-    source(f, path)
-    return i + 1
-
-def w_source(f, i):
-    path = f.stack.pop()
-    if os.path.isabs(path):
-        source(f, path)
-        return i+1
-
-    relative_dir = os.path.dirname(f.evaluate_token('*source*'))
-    relative_path = f'{relative_dir}/{path}'
-    if os.path.exists(relative_path):
-        source(f, relative_path)
-        return i+1
-
-    source(f, path)
-    return i+1
-
-
-def execute_f(name, instructions):
-    #print('execute_f:', name, len(instructions))
-    def inner(forth, i, debug=False):
-        #print('inner f:', name)
-        #print('inner f:', len(instructions))
-        j = 0
-        while j >= 0:
-            #print(j, '=>', instructions[j])
-            j = instructions[j](forth, j)
-        return i + 1
-    return inner
-
-def ifnot_jump_f(n):
-    def ifnot_jump(forth, i):
-        # print('If not jump:')
-        x = forth.stack.pop()
-        # print('==>value:', x)
-        if not x:
-            # print('==>', x, ' is false')
-            # print('==>returning', n)
-            return i+n
-        # print('==>returning 1')
-        return i+1
-    return ifnot_jump
-
-def jump_f(n):
-    def do_jump(forth, i):
-        return n+i
-    return do_jump
-
-def w_recur(f, i):
-    return 0
-
-def w_import(f, i):
-    name = f.stack.pop()
-    m = importlib.import_module(name)
-    f.namespace[name] = const_f(m)
-    return i+1
-
-def w_call(f, i):
-    func = f.stack.pop()
-    args = f.stack.pop()
-    # print('f', f, 'args', args)
-    result = func(*args)
-    # print('result', result)
-    f.stack.push(result)
-    return i+1
-
-def w_px(f, i):
-    args = f.stack.pop()
-    name = f.stack.pop()
-    m = f.stack.pop()
-    func = m.__dict__[name]
-    result = func(*args)
-    f.stack.push(result)
-    return i+1
-
-def w_unique(f, ip):  # pushes a uique object.
-    f.stack.push(Unique())
-    return ip+1
-
-def w_map(f, ip):
-    l = f.stack.pop()
-    word = f.stack.pop()
-
-    word_f = f.namespace.get(word, None)
-    result = []
-
-    for item in l:
-        f.stack.push(item)
-        word_f(f, 0)
-        result.append(f.stack.pop())
-
-    f.stack.push(result)
-    return ip+1
-
-def w_reduce(f, ip):
-    l = f.stack.pop()
-    word = f.stack.pop()
-
-    word_f = f.namespace.get(word, None)
-
-    if len(l) <= 0:
-        f.stack.push(None)
-    elif len(l) == 1:
-        f.stack.push(l[0])
-    else:
-        result = l[0]
-        l = l[1::-1]
-        for item in l:
-            f.stack.push(result)
-            f.stack.push(item)
-            word_f(f, 0)
-            result = f.stack.pop()
-        f.stack.push(result)
-
-    return ip+1
-
-def w_bounded_list(f, ip):
-    """Create a list from delimted values on the stack.
-    [list]
-    (marker a b c marker -- [a b c]
-    """
-    marker = f.stack.pop()
-    l = []
-    x = f.stack.pop()
-    while x != marker:
-        l.append(x)
-        x = f.stack.pop()
-    l.reverse()
-    f.stack.push(l)
-    return ip+1
-
-def w_to_arglist(f, ip):
-    l = f.stack.pop()
-    f.stack.push(Arglist(l))
-    return ip+1
-
-def w_list(f, ip):    # ->list
-    n = f.stack.pop()
-    l = []
-    for i in range(n):
-        l.append(f.stack.pop())
-    f.stack.push(l)
-    return ip+1
-
-def w_thread(f, i):    # @@
-    contents = f.stack.pop()
-    result = contents[0]
-    for field in contents[1::]:
-        if isinstance(field, str) and hasattr(result, field):
-            result = getattr(result, field)    # result.field
-        elif isinstance(field, Arglist):
-            result = result(*field)            # result(*field)
-        else:
-            result = result[field]             # result[field]
-    f.stack.push(result)
-    return i+1
-
-
-ListMarker = object()
-
-def w_startlist(f, i):   # [
-    f.stack.push(ListMarker)
-    return i+1
-
-def w_endlist(f, i):    # ]
-    l = []
-    x = f.stack.pop()
-    while x != ListMarker:
-        l.append(x)
-        x = f.stack.pop()
-    l.reverse()
-    f.stack.push(l)
-    return i+1
-
-MapMarker = object()
-
-def w_startmap(f, i):   # {
-    f.stack.push(MapMarker)
-    return i+1
-
-def w_endmap(f, ip):    # }
-    l = []
-    x = f.stack.pop()
-    while x != MapMarker:
-        l.append(x)
-        x = f.stack.pop()
-    if (len(l) % 2) != 0:
-        print('Maps need even number of entries.')
-        return i+1
-    l.reverse()
-    result = {}
-    for i in range(0, len(l), 2):
-        result[l[i]] = l[i+1]
-    f.stack.push(result)
-    return ip+1
-
-def w_list_to_map(f, ip):  # list->map
-    l = f.stack.pop()
-    result = {}
-    for i in range(0, len(l), 2):
-        result[l[i]] = l[i+1]
-    f.stack.push(result)
-    return ip+1
-
-def w_get(f, i):
-    name = f.stack.pop()
-    m = f.stack.pop()
-    result = m[name]
-    f.stack.push(result)
-    return i+1
-
-def w_getattribute(f, i):
-    name = f.stack.pop()
-    x = f.stack.pop()
-    result = x.__getattribute__(name)
-    f.stack.push(result)
-    return i+1
-
-def w_def(f, i):
-    value = f.stack.pop()
-    name = f.stack.pop()
-    f.defvar(name, value)
-    # print('name', name, 'value', value)
-    return i+1
-
-def w_gt(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b > a)
-    return i+1
-
-def w_lt(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b < a)
-    return i+1
-
-def w_eq(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(a==b)
-    return i+1
-
-def w_le(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b<=a)
-    return i+1
-
-def w_ge(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b>=a)
-    return i+1
-
-def w_add(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b+a)
-    return i+1
-
-def w_mul(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b*a)
-    return i+1
-
-def w_sub(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b-a)
-    return i+1
-
-def w_div(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b/a)
-    return i+1
-
-def w_reset(f, i):
-    a = f.stack.reset()
-    return i+1
-
-def w_dot(f, i):
-    a = f.stack.pop()
-    print(a, end='')
-    return i+1
-
-def w_splat(f, i):
-    l = f.stack.pop()
-    l.reverse()
-    for x in l:
-        f.stack.push(x)
-    return i+1
-
-def w_dup(f, i):
-    x = f.stack.peek()
-    f.stack.push(x)
-    return i+1
-
-def w_tmb(f, i):  # A noop
-    # t = f.stack.pop()
-    # m = f.stack.pop()
-    # b = f.stack.pop()
-    # f.stack.push(b)
-    # f.stack.push(m)
-    # f.stack.push(t)
-    return i+1
-
-def w_tbm(f, i):
-    t = f.stack.pop()
-    m = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(m)
-    f.stack.push(b)
-    f.stack.push(t)
-    return i+1
-
-def w_bmt(f, i):
-    t = f.stack.pop()
-    m = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(t)
-    f.stack.push(m)
-    f.stack.push(b)
-    return i+1
-
-def w_btm(f, i):
-    t = f.stack.pop()
-    m = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(m)
-    f.stack.push(t)
-    f.stack.push(b)
-    return i+1
-
-def w_mtb(f, i):
-    t = f.stack.pop()
-    m = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(b)
-    f.stack.push(t)
-    f.stack.push(m)
-    return i+1
-
-def w_mbt(f, i):
-    t = f.stack.pop()
-    m = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(t)
-    f.stack.push(b)
-    f.stack.push(m)
-    return i+1
-
-def w_rot(f, i):
-    c = f.stack.pop()
-    b = f.stack.pop()
-    a = f.stack.pop()
-    f.stack.push(b)
-    f.stack.push(c)
-    f.stack.push(a)
-    return i+1
-
-def w_drop(f, i):
-    f.stack.pop()
-    return i+1
-
-def w_swap(f, i):
-    a = f.stack.pop()
-    b = f.stack.pop()
-    f.stack.push(a)
-    f.stack.push(b)
-    return i+1
-
-def w_nl(f, i):
-    print()
-    return i+1
-
-def w_return(f, i):
-    return -9999;
-
-def w_colon(f, i):
-    f.compiler = Compiler()
-
-def w_semi(forth, i):
-    forth.compiler.add_instruction(w_return)
-    name = forth.compiler.name
-    word_f = execute_f(name, forth.compiler.instructions)
-    forth.namespace[name] = word_f
-    forth.compiler = None
-    return i+1
-
-w_semi.__dict__['immediate'] = True
-
-def w_should_not_happen(forth, i):
-    print('Should not execute this word!')
-    raise ValueError
-
-def w_if(forth, i):
-    #print('w_if')
-    compiler = forth.compiler
-    compiler.push_offset()
-    compiler.push_offset()
-    compiler.add_instruction(w_should_not_happen)
-    return i+1
-
-w_if.__dict__['immediate'] = True
-
-def w_then(forth, i):
-    compiler = forth.compiler
-    else_offset = compiler.pop_offset()
-    if_offset = compiler.pop_offset()
-    then_offset = compiler.offset()
-    if else_offset == if_offset:
-        delta = then_offset - if_offset
-        compiler.instructions[if_offset] = ifnot_jump_f(delta)
-    else:
-        if_delta = else_offset - if_offset + 1
-        compiler.instructions[if_offset] = ifnot_jump_f(if_delta)
-        else_delta = then_offset - else_offset
-        compiler.instructions[else_offset] = jump_f(else_delta)
-    return i+1
-
-w_then.__dict__['immediate'] = True
-
-
-def w_else(forth, i):
-    compiler = forth.compiler
-    compiler.pop_offset()
-    compiler.push_offset()
-    compiler.add_instruction(w_should_not_happen)
-    return i+1
-
-w_else.__dict__['immediate'] = True
-
-def w_do(forth, i):
-    #print('w_do')
-    compiler = forth.compiler
-    compiler.push_offset()
-    compiler.add_instruction(w_should_not_happen)
-    return i+1
-
-w_do.__dict__['immediate'] = True
-
-def w_while(forth, i):
-    compiler = forth.compiler
-    do_offset = compiler.pop_offset()
-    while_offset = compiler.offset()
-    delta = do_offset - while_offset
-    compiler.instructions[if_offset] = ifnot_jump_f(delta)
-    return i+1
-
-w_while.__dict__['immediate'] = True
-
-def w_begin(forth, i):
-    compiler = forth.compiler
-    compiler.push_offset()
-    return i+1
-
-w_begin.__dict__['immediate'] = True
-
-def w_until(forth, i):
-    compiler = forth.compiler
-    begin_offset = compiler.pop_offset()
-    until_offset = compiler.offset()
-    delta = begin_offset - until_offset
-    #print('Delta:', delta)
-    compiler.instructions.append(ifnot_jump_f(delta))
-    return i+1
-
-
-w_until.__dict__['immediate'] = True
-
-
-def w_dump(f, i):
-    f.dump()
-    return i+1
-
-def w_idump(f, i):
-    f.dump()
-    return i+1
-
-w_idump.__dict__['immediate'] = True
-
-def w_stack(f, i):
-    print("Stack:", end=' ')
-    for x in f.stack:
-        print(f'{repr(x)}', end=' ')
-    print()
-    return i+1