Rationalize namespaces by adding entries. Add readtoken which has different behavior at compile and runtime.

sallyforth/basic_words.py

@@ -29,10 +29,10 @@ def import_native_module(forth, m, alias=None, excludes=[]):
     names = [x for x in raw_names if x not in excludes] 
     for name in names:
         localname = f'{alias}.{name}'
-        val = m.__getattribute__(name)
+        val = getattr(m, name)
-        forth.namespace[localname] = const_f(val)
+        forth.namespace.set(localname, const_f(val))
 
-def w_nexttoken(f, i):
+def xx_w_nexttoken(f, i):
     token = f.read_next_token()
     f.stack.push(token)
     return i+1
@@ -139,7 +139,7 @@ def w_recur(f, i):
 def w_import(f, i):
     name = f.stack.pop()
     m = importlib.import_module(name)
-    f.namespace[name] = const_f(m)
+    f.namespace.set(name, const_f(m))
     return i+1
 
 def w_call(f, i):
@@ -165,20 +165,38 @@ def w_return(f, i):
 def w_colon(f, i):
     f.compiler = Compiler()
 
-def w_semi(forth, i):
+def i_semi(forth, i):
     forth.compiler.add_instruction(w_return)
     name = forth.compiler.name
     word_f = execute_f(name, forth.compiler.instructions)
-    forth.namespace[name] = word_f
+    forth.defword(name, word_f)
     forth.compiler = None
     return i+1
-w_semi.__dict__['immediate'] = True
+
+def w_compiling(f, i):
+    return f.stack.push(f.compiling())
+
+def i_readtoken(f, i):
+    kind, token = f.read_next_token()
+    if f.compiling():
+        compiler = f.compiler
+        compiler.add_instruction(const_f(token))
+    else:
+        f.stack.push(token)
+    return i+1
+
+def w_immediate(f, i):
+    flag = f.stack.pop()
+    name = f.stack.pop()
+    print(f'name: {name} flag {flag}')
+    f.namespace[name].immediate = flag
+    return i+1
 
 def w_should_not_happen(forth, i):
     print('Should not execute this word!')
     raise ValueError
 
-def w_if(forth, i):
+def i_if(forth, i):
     #print('w_if')
     compiler = forth.compiler
     compiler.push_offset()
@@ -186,9 +204,7 @@ def w_if(forth, i):
     compiler.add_instruction(w_should_not_happen)
     return i+1
 
-w_if.__dict__['immediate'] = True
+def i_then(forth, i):
-
-def w_then(forth, i):
     compiler = forth.compiler
     else_offset = compiler.pop_offset()
     if_offset = compiler.pop_offset()
@@ -203,28 +219,21 @@ def w_then(forth, i):
         compiler.instructions[else_offset] = jump_f(else_delta)
     return i+1
 
-w_then.__dict__['immediate'] = True
+def i_else(forth, i):
-
-
-def w_else(forth, i):
     compiler = forth.compiler
     compiler.pop_offset()
     compiler.push_offset()
     compiler.add_instruction(w_should_not_happen)
     return i+1
 
-w_else.__dict__['immediate'] = True
+def i_do(forth, i):
-
-def w_do(forth, i):
     #print('w_do')
     compiler = forth.compiler
     compiler.push_offset()
     compiler.add_instruction(w_should_not_happen)
     return i+1
 
-w_do.__dict__['immediate'] = True
+def i_while(forth, i):
-
-def w_while(forth, i):
     compiler = forth.compiler
     do_offset = compiler.pop_offset()
     while_offset = compiler.offset()
@@ -232,16 +241,12 @@ def w_while(forth, i):
     compiler.instructions[if_offset] = ifnot_jump_f(delta)
     return i+1
 
-w_while.__dict__['immediate'] = True
+def i_begin(forth, i):
-
-def w_begin(forth, i):
     compiler = forth.compiler
     compiler.push_offset()
     return i+1
 
-w_begin.__dict__['immediate'] = True
+def i_until(forth, i):
-
-def w_until(forth, i):
     compiler = forth.compiler
     begin_offset = compiler.pop_offset()
     until_offset = compiler.offset()
@@ -250,19 +255,14 @@ def w_until(forth, i):
     compiler.instructions.append(ifnot_jump_f(delta))
     return i+1
 
-
-w_until.__dict__['immediate'] = True
-
 def w_dump(f, i):
     f.dump()
     return i+1
 
-def w_idump(f, i):
+def i_idump(f, i):
     f.dump()
     return i+1
 
-w_idump.__dict__['immediate'] = True
-
 def w_stack(f, i):
     print("Stack:", end=' ')
     for x in f.stack:

sallyforth/data_words.py

@@ -1,4 +1,5 @@
 from unique import Unique
+from arglist import Arglist
 
 def w_unique(f, ip):  # pushes a uique object.
     f.stack.push(Unique())
@@ -8,7 +9,7 @@ def w_map(f, ip):
     word = f.stack.pop()
     l = f.stack.pop()
 
-    word_f = f.namespace.get(word, None)
+    word_f = f.namespace.get(word, None).get_ivalue()
     result = []
 
     for item in l:
@@ -23,7 +24,7 @@ def w_reduce(f, ip):
     l = f.stack.pop()
     word = f.stack.pop()
 
-    word_f = f.namespace.get(word, None)
+    word_f = f.namespace.get(word, None).get_ivalue()
 
     if len(l) <= 0:
         f.stack.push(None)

sallyforth/init.sf

@@ -32,16 +32,18 @@
 : expanduser #os.path.expanduser !!1 ;
 : expand expanduser expandvars ;
 
-\: prompt-and-run ( -- prog-status)
+: tokenize <. $? 'split .> " " swap !!1 ;
-\  ">> " read-line
+
-\  dup "x" = 
+: prompt-and-run ( -- prog-status)
-\  if
+  ">> " read-line
-\    "Exit!" p
+  dup "x" = 
-\    drop
+  if
-\  else
+    "Exit!" p
-\    tokenize
+    drop
-\   'expand map
+  else
-\    run-prog
+    tokenize
-\    recur
+   'expand map
-\  then
+    run-prog
-\;
+    recur
+  then
+;

sallyforth/io.sf

@@ -9,4 +9,4 @@
   builtins.input !!1
 ;
 
-: read-next nexttoken second ;
+\ : read-next nexttoken second ;

sallyforth/kernel.py

@@ -2,7 +2,6 @@ import sys
 from os import path
 import basic_words, data_words, operator_words, stack_words, os_words
 from basic_words import const_f, w_enlist
-#from lex import  is_string, Tokenizer
 import tokenstream as ts
 from stack import Stack
 from namespace import Namespace
@@ -21,26 +20,25 @@ class Forth:
         self.streams = Stack()
         self.stack = Stack()
         self.namespaces = {}
-        initial_defs = {
-                '*prompt*': const_f('SallyForth>> '),
-                'macroexpand': w_enlist,
-                '*source*': const_f(__file__),
-                'true': const_f(True),
-                'false': const_f(False),
-                'None': const_f(None),
-                '0': const_f(0),
-                '1': const_f(1),
-                '2': const_f(2)}
 
-        self.forth_ns = self.make_namespace('forth', initial_defs)
+        self.forth_ns = self.make_namespace('forth')
+        self.namespace = self.forth_ns
         user_ns = self.make_namespace('user', {}, [self.forth_ns])
 
-        self.forth_ns.import_from_module(basic_words, 'w_')
+        self.defword('*prompt*', const_f('SallyForth>> '))
-        self.forth_ns.import_from_module(data_words, 'w_')
+        self.defword('*source*', const_f(__file__))
-        self.forth_ns.import_from_module(operator_words, 'w_')
+        self.defword('true', const_f(True))
-        self.forth_ns.import_from_module(stack_words, 'w_')
+        self.defword('false', const_f(False))
-        self.forth_ns.import_from_module(os_words, 'w_')
+        self.defword('None', const_f(None))
-        self.namespace = self.forth_ns
+        self.defword('0', const_f(0))
+        self.defword('1', const_f(1))
+        self.defword('2', const_f(2))
+
+        self.forth_ns.import_from_module(basic_words)
+        self.forth_ns.import_from_module(data_words)
+        self.forth_ns.import_from_module(operator_words)
+        self.forth_ns.import_from_module(stack_words)
+        self.forth_ns.import_from_module(os_words)
 
         self.compiler = None
 
@@ -51,15 +49,19 @@ class Forth:
 
         self.namespace = user_ns
 
+    def defword(self, name, value):
+        self.namespace.set(name, value)
+
     def defvar(self, name, value):
-        self.namespace[name] = const_f(value)
+        self.defword(name, const_f(value))
 
     def compiling(self):
         return self.compiler
 
     def _compile_token(self, kind, token):
-        print(f"compile: {self.compiler.name}: {kind} {token}")
+        #print(f"compile: {self.compiler.name}: {token}")
         if self.compiler.name == None:
+            print(f'Compiling {token}')
             self.compiler.name = token
             return
 
@@ -68,11 +70,14 @@ class Forth:
             return
 
         if token in self.namespace:
-            word = self.namespace[token]
+            entry = self.namespace[token]
-            if 'immediate' in word.__dict__:
+            #print(token, entry)
-                word(self, 0)
+            if entry.immediate:
+                value = entry.get_ivalue()
+                value(self, 0)
             else:
-                self.compiler.add_instruction(self.namespace[token])
+                value = entry.get_cvalue()
+                self.compiler.add_instruction(value)
             return
 
         n = to_number(token)
@@ -88,8 +93,10 @@ class Forth:
             return
 
         if token in self.namespace:
-            # print("executing ", token)
+            #print("executing ", token)
-            self.namespace[token](self, 0)
+            f = self.namespace[token].get_ivalue()
+            #print(f)
+            f(self, 0)
             return
 
         n = to_number(token)
@@ -99,26 +106,22 @@ class Forth:
             self.stack.push(n)
 
     def execute_token(self, kind, token):
-        #print(f'execute kind {kind} token: {token}')
+        #print(f'execute_token: {token}')
         kts = self.macro_expand_token(kind, token)
         #print(kts)
         for kt in kts:
             this_kind, this_token = kt
-            #print(f'execute this {this_kind} {this_token}')
             if not self.compiling():
+                #print("interactive", this_token)
                 self._eval_token(this_kind, this_token)
             else:
+                #print("compiling...", this_token)
                 self._compile_token(this_kind, this_token)
-
+        #print("Done")
-    def XX_execute_token_stream(self, s):
-        kind, token = s.get_token()
-        while kind != 'eof':
-            self.execute_token(kind, token)
-            kind, token = s.get_token()
 
     def execute_current_stream(self):
         s = self.streams.peek()
-        print("exec current s:", s)
+        #print("exec current s:", s)
         kind, token = s.get_token()
         while kind != 'eof':
             self.execute_token(kind, token)
@@ -126,7 +129,7 @@ class Forth:
         self.streams.pop()
 
     def execute_token_stream(self, s):
-        print("exec token stream:", s)
+        #print("exec token stream:", s)
         self.streams.push(s)
         self.execute_current_stream()
 
@@ -143,7 +146,7 @@ class Forth:
         return s.get_token()
 
     def py_evaluate(self, s, *args):
-        print(f'Evaluate: token [{token}] args <<{args}>>')
+        #print(f'Evaluate: token [{token}] args <<{args}>>')
         rargs = list(args)
         rargs.reverse()
         if rargs:

sallyforth/namespace.py

@@ -1,3 +1,18 @@
+class Entry:
+    def __init__(self, name, value, immediate):
+        self.name = name
+        self.value = value
+        self.immediate = immediate
+
+    def get_ivalue(self):
+        return self.value
+
+    def get_cvalue(self):
+        return self.value
+
+    def __str__(self):
+        return f'Entry {self.name} {self.immediate}'
+
 class Namespace:
     def __init__(self, name, initial_contents={}, refers=[]):
         self.name = name
@@ -10,18 +25,32 @@ class Namespace:
         """
         self.refers.append(ns)
 
-    def import_from_module(self, m, prefix):
+    def import_from_module(self, m):
         """
         Import all of the word defining functions in
         module m whose function names start with prefix
         into this namespace. Removes the prefix.
         """
         names = dir(m)
-        prefix_len = len(prefix)
         for name in names:
-            if name.startswith(prefix):
+            if name.startswith("w_"):
-                word_name = name[prefix_len::]
+                word_name = name[2::]
-                self[word_name] = m.__getattribute__(name)
+                #print(f'Setting {word_name} to false')
+                self.set(word_name, getattr(m, name))
+            elif name.startswith("i_"):
+                word_name = name[2::]
+                #print(f'Setting {word_name} to true')
+                self.set(word_name, getattr(m, name), immediate=True)
+
+    def set(self, name, value, cvalue=None, immediate=False):
+        if name not in self.contents:
+            entry = Entry(name, value, immediate)
+        else:
+            entry = self[name]
+            entry.value = value
+            entry.cvalue = cvalue
+            entry.immediate = immediate
+        self.contents[name] = entry
 
     def keys(self):
         return self.contents.keys()

sallyforth/os_words.py

@@ -4,8 +4,8 @@ import sys
 def w_fork(f, i):
     parent_word = f.stack.pop()
     child_word = f.stack.pop()
-    parent_f = f.namespace.get(parent_word, None)
+    parent_f = f.namespace.get(parent_word, None).get_ivalue()
-    child_f = f.namespace.get(child_word, None)
+    child_f = f.namespace.get(child_word, None).get_ivalue()
     pid = os.fork()
     f.stack.push(pid)
     if pid == 0:

sallyforth/sallyforth.py

@@ -31,9 +31,6 @@ def setup_readline(history_path, f):
     def save_history():
         readline.write_history_file(history_path)
     atexit.register(save_history)
-    def prompt_f():
-        return f.evaluate_string('*prompt*')
-    return prompt_token_stream(prompt_f)    
 
 def setup_forth():
     source_dir = os.path.dirname(os.path.abspath(__file__))
@@ -45,11 +42,21 @@ def setup_forth():
         f = Forth()
     return f
 
-def repl(stream, f):
+def repl(f):
-    f.execute_token_stream(stream)
+    while True:
+        prompt = f.evaluate_string('*prompt*')
+        try:
+            line = input(prompt)
+        except EOFError:
+            return
+        try:
+            f.execute_string(line)
+        except:
+            traceback.print_exc()
+
 
 if __name__ == "__main__":
     f = setup_forth()
-    stream = setup_readline(hist_file, f)
+    setup_readline(hist_file, f)
-    repl(stream, f)
+    repl(f)
     print("Bye!")

sallyforth/stack.py

@@ -16,6 +16,7 @@ class Stack:
         if self.top < -1:
             print("stack overpop")
             self.top = -1;
+            raise ValueError("Stack underflow")
         return result
 
     def __iter__(self):