Break up basic_words a bit,add documentation to both native and sallyforth words.

sallyforth/0.sf

@@ -19,6 +19,7 @@
 : ->static   { false *last-word* dynamic }
 : ->dynamic  { true *last-word* dynamic }
 
+
 : -- { 1 - }           ->compile
 : ++ { 1 + }           ->compile
 : =0 { 0 = }           ->compile
@@ -37,9 +38,14 @@ unique 'list-marker =!
 : [] { [ ] }                ->compile
 
 unique 'map-marker =!
-: <<= map-marker                       ->compile
-: =>> { map-marker [list] list->map }  ->compile
-: <<=>> { <<= =>> }                    ->compile
+
+: <<= /Start a map/ map-marker ->compile
+
+: =>> /Complete a map/ {
+  map-marker [list] list->map 
+} ->compile
+
+: <<=>> /Push an empty map/ { <<= =>> } ->compile
 
 
 / Spelunk thru objects and properties.
@@ -51,9 +57,9 @@ unique 'map-marker =!
 
 / Function calling.
 
-: !!0 { [] swap !! }
-: !!1 { swap 1 ->list swap !! }
-: !!2 { mbt 2 ->list swap !! }
+: !!0 /Call a 0 arg native function/ { [] swap !! }
+: !!1 /Call a 1 arg native function/ { swap 1 ->list swap !! }
+: !!2 /Call a 2 arg native function/ { mbt 2 ->list swap !! }
 
 / obj attr -- attr-value 
 : getattr  {
@@ -109,6 +115,11 @@ unique 'map-marker =!
                  { "Namespace " .  . " loaded." p }  
 }
   
+/ Documentartion
+
+: ->doc { *last-word* setdoc }
+: __doc__ { <. $? '__doc__ .> }
+
 / Other startup files.
 
 *sallyforth-dir* "/"  "io.sf"     + + source

sallyforth/basic_words.py

@@ -1,79 +1,18 @@
-import tokenstream as ts
-from wrappers import noop
-from namespace import Namespace
-from util import word, native_word
+from util import word
 from unique import Unique
-import python_compiler as pc
-import inliner
-import importlib
 from pprint import pprint
 
-@word()
-def compile(forth):
-    name = forth.stack.pop()
-    var = forth.ns[name]
-    word_f = var.value
-    new_f = pc.compile_word_f(word_f, name)
-    forth.set(name, new_f)
-
-@word()
-def inline(forth):
-    name = forth.stack.pop()
-    var = forth.ns[name]
-    word_f = var.value
-    new_f = inliner.compile_word_f(word_f, name)
-    forth.set(name, new_f)
-
-@word()
-def dynamic(forth):
-    name = forth.stack.pop()
-    isdyn = forth.stack.pop()
-    var = forth.ns[name]
-    var.dynamic = isdyn
-
-@word()
-def native(forth):
-    has_return = forth.stack.pop()
-    n = forth.stack.pop()
-    native_f = forth.stack.pop()
-    name = forth.stack.pop()
-    print('has_return', has_return)
-    print('n', n)
-    print('native_f', native_f)
-    print('name', name)
-    wrapped_f = native_word(native_f, name, n, has_return)
-    forth.set(name, wrapped_f)
-
-@word("go!")
+@word("!", doc='Execute the word from the stack: word -- <results>')
 def exec_word(forth):
     func = forth.stack.pop()
     func(forth)
 
-@word("function")
-def function_word(forth):
-    name = forth.stack.pop()
-    var = forth.ns[name]
-    word_f = var.value
-    def native_f(*args):
-        forth.stack.push(args)
-        word_f(forth)
-        result = forth.stack.pop()
-        return result
-    forth.stack.push(native_f)
-
 @word('raise')
 def w_raise(forth):
     ex = forth.stack.pop()
     raise ex
 
-@word(immediate=True)
-def readtoken(forth):
-    t = forth.stream.get_token()
-    def push_token(xforth):
-        xforth.stack.push(t)
-    return push_token
-
-@word("!!")
+@word("!!", doc='Execute a raw function from the stack: func -- <results>')
 def w_call(forth):
     func = forth.stack.pop()
     args = forth.stack.pop()
@@ -84,129 +23,37 @@ def w_call(forth):
         raise
     forth.stack.push(result)
 
-@word()
+@word(doc='Push a new unique value onto the stack: -- unique')
 def unique(forth):
     forth.stack.push(Unique())
 
-@word()
-def load(forth):
-    name = forth.stack.pop()
-    m = importlib.import_module(name)
-    forth.set_constant(name, m)
-
-@word('import')
-def w_import(forth):
-    name = forth.stack.pop()
-    m = importlib.import_module(name)
-    forth.ns.import_native_module(m)
-
-@word()
-def lexicon(forth):
-    name = forth.stack.pop()
-    forth.ns.import_from_module(name)
-
-@word('source')
-def w_source(forth):
-    path = forth.stack.pop()
-    forth.eval_file(path)
-
-@word('alias')
-def w_alias(forth):
-    new_name = forth.stack.pop()
-    old_name = forth.stack.pop()
-    forth.alias(new_name, old_name)
-
-@word()
-def rawdef(forth):
-    name = forth.stack.pop()
-    value = forth.stack.pop()
-    forth.set(name, value)
-
-@word("=!")
-def equal_bang(forth):
-    name = forth.stack.pop()
-    value = forth.stack.pop()
-    forth.set_constant(name, value)
-
 @word("*prompt*")
 def promptword(forth):
     forth.stack.push(">> ")
 
-@word()
-def lookup(forth):
-    name = forth.stack.pop()
-    forth.stack.push(forth.ns[name])
-
-@word()
-def forget(forth):
-    name = forth.stack.pop()
-    del forth.ns[name]
-
 @word()
 def p(forth):
     print(forth.stack.pop())
 
-@word()
+@word(doc='Print a newline: - ')
 def nl(forth):
     print()
 
-@word('.')
+@word('.', doc='Print the value on top of the stack: v --')
 def dot(forth):
     print(forth.stack.pop(), end='')
 
 @word()
-def splat(forth):
+def splat(forth, doc='Pop a collection and push each item separately: col -- col[n]..col[0]'):
     l = forth.stack.pop()
     l.reverse()
     for x in l:
         forth.stack.push(x)
 
-@word()
+@word(doc='Print the stack: --')
 def stack(forth):
     print(forth.stack)
 
-@word('debug-ns')
-def debug_ns(forth):
-    print('debug ns')
-    print(forth.ns.name)
-    pprint(forth.ns.includes)
-    pprint(forth.ns.contents)
-
-@word('*ns*')
-def star_ns_star(forth):
-    forth.stack.push(forth.ns)
-
-@word('new-ns')
-def new_ns(forth):
-    name = forth.stack.pop()
-    core = forth.namespaces['core']
-    namespace = Namespace(name, [core])
-    forth.namespaces[name] = namespace
-
-@word('include')
-def include_ns(forth):
-    name = forth.stack.pop()
-    included = forth.namespaces[name]
-    forth.ns.include_ns(included)
-
-@word('set-ns')
-def set_ns_word(forth):
-    name = forth.stack.pop()
-    forth.set_ns(name)
-
-@word('ns?')
-def ns_question(forth):
-    name = forth.stack.pop()
-    forth.stack.push(name in forth.namespaces)
-
-@word(':', True)
-def colon(forth):
-    name = forth.stream.get_token().value
-    body = forth.compile_next()
-    forth.set(name, body)
-    forth.core.set_constant('*last-word*', name)
-    return noop
-
 @word()
 def current_stream(forth):
     forth.stack.push(forth.stream)

sallyforth/inliner.py

@@ -1,6 +1,6 @@
 from wrappers import inner_f
 
-def compile_f(contents, name):
+def compile_f(contents, attributes, doc, name):
     new_contents = []
     for f in contents:
         sub_contents = getattr(f, "contents", None)
@@ -9,11 +9,14 @@ def compile_f(contents, name):
         else:
             new_contents.append(f)
     new_func = inner_f(new_contents)
+    if attributes:
+        new_func.__dict__ = attributes.copy()
+    new_func.__doc__ = doc
     new_func.name = name
     return new_func
 
 def compile_word_f(f, name=None):
     contents = getattr(f, 'contents', None)
     if contents and len(contents) > 1:
-        return compile_f(contents, name)
+        return compile_f(contents, f.__dict__, f.__doc__, name)
     return f

sallyforth/kernel.py

@@ -2,10 +2,7 @@ import sys
 import os
 from stack import Stack
 from namespace import Namespace
-#import basic_words
-#import stack_words
-#import operator_words
-#import data_words
+from kword import Keyword
 import tokenstream as ts
 import threaded_compiler as compiler
 from wrappers import value_f
@@ -54,6 +51,7 @@ class Forth:
         sally_dir = os.path.dirname(os.path.abspath(__file__))
         self.set_constant('*sallyforth-dir*', sally_dir)
         self.ns.import_from_module('basic_words')
+        self.ns.import_from_module('namespace_words')
         self.ns.import_from_module('stack_words')
         self.ns.import_from_module('operator_words')
         self.ns.import_from_module('data_words')
@@ -94,7 +92,8 @@ class Forth:
 
     def compile_next(self, current_token=None):
         """
-        Compile the next token, either the one passed in or the next one on the current token stream.
+        Compile the next token, either the one passed in 
+        or the next one on the current token stream.
         """
         return compiler.compile_next(self, self.stream, current_token)
 
@@ -133,6 +132,19 @@ class Forth:
         self.eval_string(s)
         return self.stack.pop()
 
+    def eval_object(self, o):
+        if isinstance(o, [int, str, Keyword, float]):
+            self.stack.push(o)
+        elif callable(o):
+            o(self)
+        else:
+            print(o, "??")
+            raise ValueError()
+
+    def eval_objects(self, l):
+        for o in l:
+            self.eval_object(l)
+
     def lookup(self, name):
         """
         Return the value of the given name in the current namespace.

sallyforth/kword.py

@@ -10,8 +10,9 @@ class Keyword(UserString):
         value = value[1::]
         UserString.__init__(self, value)
 
-    def __call__(self, d):
-        return d[self]
+    def __call__(self, forth):
+        d = forth.stack.pop()
+        forth.stack.push(d[self])
 
     def __repr__(self):
         return ':' + str(self)

sallyforth/list.sf

@@ -30,7 +30,7 @@
   1 ->list * 
 } ->compile
 
-: len { builtins/len !!1 }
+: len { builtins/len !!1 } builtins/len __doc__ ->doc
 
 : empty? { len zero? }
   
@@ -38,8 +38,8 @@
 : rrest  { rest rest }      ->compile
 : rrrest { rest rest rest } ->compile
 
-: ffirst  { first first }       ->compile
-: fffirst { first first first } ->compile
+: ffirst  /l -- first-item-of-first-item/ { first first }       ->compile
+: fffirst /l -- f-of-f-of-f-of-f/ { first first first } ->compile
 
 : append  {
   dup tbm 

sallyforth/namespace.py

@@ -64,9 +64,7 @@ class Namespace:
 
     def import_from_module(self, module_name):
         """
-        Import all of the word defining functions in
-        module m whose function names start with prefix
-        into this namespace. Removes the prefix.
+        Import all of the word defining functions in module m.
         """
         m = load_module(module_name)
         names = dir(m)
@@ -74,6 +72,8 @@ class Namespace:
             value = getattr(m, name)
             if get_attribute(value, 'forth_word'):
                 forth_name = value.forth_name or name
+                if forth_name in self:
+                    print("Warning: redefining", forth_name)
                 var = self.set(forth_name, value, False)
 
     def import_native_module(self, m, alias=None):
@@ -120,8 +120,8 @@ class Namespace:
                 return True
         return False
 
-    def __delattr__(self, key):
-        return self.contents.__delattr__(key)
+    def __delitem__(self, key):
+        return self.contents.__delitem__(key)
 
     def __setitem__(self, key, x):
         return self.set(key, x)

sallyforth/namespace_words.py

@@ -0,0 +1,174 @@
+from wrappers import noop
+from namespace import Namespace
+from util import word, native_word
+import python_compiler as pc
+import inliner
+import importlib
+from pprint import pprint
+
+@word(doc='Given a name or a word, return the docstring: name-or-word -- docstr')
+def doc(forth):
+    f = forth.stack.pop()
+    if not callable(f):
+        f = forth.ns[f].value
+    if hasattr(f, '__doc__'):
+        forth.stack.push(f.__doc__)
+    else:
+        forth.stack.push('')
+
+@word(doc='Compile forth word: name -- ')
+def compile(forth):
+    name = forth.stack.pop()
+    var = forth.ns[name]
+    word_f = var.value
+    new_f = pc.compile_word_f(word_f, name)
+    forth.set(name, new_f)
+
+@word(doc='Expand forth word: name -- ')
+def inline(forth):
+    name = forth.stack.pop()
+    var = forth.ns[name]
+    word_f = var.value
+    new_f = inliner.compile_word_f(word_f, name)
+    forth.set(name, new_f)
+
+@word()
+def dynamic(forth):
+    name = forth.stack.pop()
+    isdyn = forth.stack.pop()
+    var = forth.ns[name]
+    var.dynamic = isdyn
+
+@word()
+def native(forth):
+    has_return = forth.stack.pop()
+    n = forth.stack.pop()
+    native_f = forth.stack.pop()
+    name = forth.stack.pop()
+    wrapped_f = native_word(native_f, name, n, has_return)
+    forth.set(name, wrapped_f)
+
+@word("function")
+def function_word(forth):
+    name = forth.stack.pop()
+    var = forth.ns[name]
+    word_f = var.value
+    def native_f(*args):
+        forth.stack.push(args)
+        word_f(forth)
+        result = forth.stack.pop()
+        return result
+    forth.stack.push(native_f)
+
+@word(immediate=True)
+def readtoken(forth):
+    t = forth.stream.get_token()
+    def push_token(xforth):
+        xforth.stack.push(t)
+    return push_token
+
+@word(doc='Load a new native module: modname --')
+def load(forth):
+    name = forth.stack.pop()
+    m = importlib.import_module(name)
+    forth.set_constant(name, m)
+
+@word('import', doc='Import a native module, bind all of the module values: modname --')
+def w_import(forth):
+    name = forth.stack.pop()
+    m = importlib.import_module(name)
+    forth.ns.import_native_module(m)
+
+@word(doc='Import a module that defines forth words: modname -- ')
+def lexicon(forth):
+    name = forth.stack.pop()
+    forth.ns.import_from_module(name)
+
+@word('source', doc='Read an execute a file full of forth code: path --')
+def w_source(forth):
+    path = forth.stack.pop()
+    forth.eval_file(path)
+
+@word('alias')
+def w_alias(forth):
+    new_name = forth.stack.pop()
+    old_name = forth.stack.pop()
+    forth.alias(new_name, old_name)
+
+@word("=!")
+def equal_bang(forth):
+    name = forth.stack.pop()
+    value = forth.stack.pop()
+    forth.set_constant(name, value)
+
+@word()
+def rawdef(forth):
+    name = forth.stack.pop()
+    value = forth.stack.pop()
+    forth.set(name, value)
+
+@word()
+def lookup(forth):
+    name = forth.stack.pop()
+    forth.stack.push(forth.ns[name])
+
+@word(doc='Forget the word from the stack: wordname --')
+def forget(forth):
+    name = forth.stack.pop()
+    del forth.ns[name]
+
+@word('debug-ns')
+def debug_ns(forth):
+    print('debug ns')
+    print(forth.ns.name)
+    pprint(forth.ns.includes)
+    pprint(forth.ns.contents)
+
+@word('*ns*')
+def star_ns_star(forth):
+    forth.stack.push(forth.ns)
+
+@word('new-ns')
+def new_ns(forth):
+    name = forth.stack.pop()
+    core = forth.namespaces['core']
+    namespace = Namespace(name, [core])
+    forth.namespaces[name] = namespace
+
+@word('include')
+def include_ns(forth):
+    name = forth.stack.pop()
+    included = forth.namespaces[name]
+    forth.ns.include_ns(included)
+
+@word('set-ns')
+def set_ns_word(forth):
+    name = forth.stack.pop()
+    forth.set_ns(name)
+
+@word('ns?')
+def ns_question(forth):
+    name = forth.stack.pop()
+    forth.stack.push(name in forth.namespaces)
+
+@word(':', True)
+def colon(forth):
+    name = forth.stream.get_token().value
+    tok = forth.stream.get_token(True)
+    docstring = None
+    if tok.iscomment():
+        docstring = tok.value
+        tok = None
+    body = forth.compile_next(tok)
+    if docstring:
+        body.__doc__ = docstring
+    forth.set(name, body)
+    forth.core.set_constant('*last-word*', name)
+    return noop
+
+@word()
+def setdoc(forth):
+    word = forth.stack.pop()
+    doc = forth.stack.pop()
+    f = forth.ns[word].value
+    f.__doc__ = doc

sallyforth/python_compiler.py

@@ -26,7 +26,7 @@ def print_ast(name):
                 keywords=[]))
     return r
 
-def compile_f(contents, name):
+def compile_f(contents, attributes, doc, name):
     d = locals().copy()
     exprs = []
     for i, val in enumerate(contents):
@@ -39,7 +39,9 @@ def compile_f(contents, name):
     code = compile(m, 'source', 'exec')
     exec(code, d)
     f = d['generated_function']
-    f.immediate = False
+    if attributes:
+        f.__dict__ = attributes.copy()
+    f.__doc__ = doc
     f.operation_type = 'compiled'
     f.name = name
     f.contents = contents
@@ -57,5 +59,5 @@ def compile_word_f(f, name=None):
     """
     contents = getattr(f, 'contents', None)
     if contents and len(contents) > 1:
-        return compile_f(contents, name)
+        return compile_f(contents, f.__dict__, f.__doc__, name)
     return f

sallyforth/tokenstream.py

@@ -1,4 +1,5 @@
 import io
+from kword import Keyword
 
 def to_number(token):
     try:
@@ -41,6 +42,9 @@ class Token:
     def isnumber(self):
         return self.kind == 'number'
 
+    def iscomment(self):
+        return self.kind == 'comment'
+
     def __repr__(self):
         return str(self)
 
@@ -90,8 +94,15 @@ class TokenStream:
     def ender(self, ch):
         return self.whitespace(ch) or self.special(ch)
 
-    def get_token(self):
+    def get_token(self, return_comment=False):
+        #print("ret comment:", return_comment)
+        if return_comment:
+            return self.do_get_token()
+
         t = self.do_get_token()
+        while t and t.iscomment():
+            t = self.do_get_token()
+
         return t
 
     def next_ch(self):
@@ -121,24 +132,26 @@ class TokenStream:
                     return Token('string', token)
                 if state in ['word']:
                     return Token('word', token)
+                if state == 'comment':
+                    return Token('comment', token)
+                if state == 'keyword':
+                    return Token('keyword', Keyword(token))
                 if state == 'number':
                     return self.number_or_word(token)
                 return None
             elif state == 'start' and self.special(ch):
                 return Token('word', ch)
-
             elif state == 'start' and ch == ':':
                 token = ch
                 state = 'keyword'
             elif state == 'start' and ch in "+-0123456789":
                 token = ch
                 state = 'number'
-            elif state == 'lcomment' and ch == '\n':
-                state = 'start'
             elif state == 'start' and ch == '/':
-                state = 'icomment'
-            elif state == 'icomment' and ch in ['\n', '/']:
-                state = 'start'
+                token = ''
+                state = 'comment'
+            elif state == 'comment' and ch in ['\n', '/']:
+                return Token('comment', token) 
             elif state == 'start' and self.whitespace(ch):
                 continue
             elif state == 'start' and ch == '"':
@@ -163,8 +176,8 @@ class TokenStream:
                 self.unread(ch)
                 if token in [':']:
                     return Token('word', token)
-                return Token('keyword', token)
-            elif state in ['word', 'dqstring', 'sqstring', 'number', 'keyword']:
+                return Token('keyword', Keyword(token))
+            elif state in ['word', 'dqstring', 'sqstring', 'number', 'keyword', 'comment']:
                 token += ch
 
 class MacroTokenStream:
@@ -177,8 +190,8 @@ class MacroTokenStream:
         self.stream = stream
         self.tokens = []
 
-    def get_more_tokens(self):
-        raw_token = self.stream.get_token()
+    def get_more_tokens(self, return_comment):
+        raw_token = self.stream.get_token(return_comment)
         if raw_token \
            and raw_token.isword() \
            and raw_token.value[0] == '#':
@@ -192,9 +205,9 @@ class MacroTokenStream:
         else:
             self.tokens.append(raw_token)
 
-    def get_token(self):
+    def get_token(self, return_comment=False):
         if len(self.tokens) == 0:
-            self.get_more_tokens()
+            self.get_more_tokens(return_comment)
         if len(self.tokens):
             return self.tokens.pop()
         return None
@@ -220,7 +233,8 @@ if __name__ == "__main__":
     pis = PromptInputStream(pmt)
     ts = TokenStream(pis.getc)
     
-    result = ts.get_token()
+    result = ts.get_token(True)
+    #print("result", result)
     while result:
         print("result:", result)
-        result = ts.get_token()
+        result = ts.get_token(True)

sallyforth/util.py

@@ -2,9 +2,10 @@ def get_attribute(x, name):
     return getattr(x, name, None)
 
 class word:
-    def __init__(self, name=None, immediate=False):
+    def __init__(self, name=None, immediate=False, doc=None):
         self.name = name
         self.immediate = immediate
+        self.doc = doc
 
     def __call__(self, f):
         f.forth_word = True
@@ -12,6 +13,8 @@ class word:
             f.forth_name = self.name
         else:
             f.forth_name = f.__name__
+        if self.doc:
+            f.__doc__ = self.doc
         f.immediate = self.immediate
         return f
 

sallyforth/wrappers.py

@@ -7,6 +7,7 @@ def value_f(value):
     push_constant.immediate = False
     push_constant.operation_type = 'pushv'
     push_constant.value = value
+    push_constant.__doc__ = value.__doc__
     return push_constant
 
 def inner_f(contents):