Reorganize python word impls, switch tokenizer to a stream approach.

This commit is contained in:
Russ Olsen 2020-04-29 14:16:54 -04:00
parent 638d50b100
commit f303b00345
17 changed files with 934 additions and 700 deletions

280
sallyforth/basic_words.py Normal file
View file

@ -0,0 +1,280 @@
from inspect import isfunction, isbuiltin
import importlib
import os
from compiler import Compiler
from arglist import Arglist
def const_f(value):
def x(f, i):
#print("const f, pushing", value)
f.stack.push(value)
return i + 1
return x
def native_function_handler(func):
def handle(forth, i):
args = forth.stack.pop()
#print(f"Native fun, calling {func}({args})")
result = func(*args)
#print(f'Result: {result}')
forth.stack.push(result)
#print("pushed result")
return i + 1
return handle
def import_native_module(forth, m, alias=None, excludes=[]):
if not alias:
alias = m.__name__
raw_names = dir(m)
names = [x for x in raw_names if x not in excludes]
for name in names:
localname = f'{alias}.{name}'
val = m.__getattribute__(name)
forth.namespace[localname] = const_f(val)
def w_nexttoken(f, i):
token = f.read_next_token()
f.stack.push(token)
return i+1
def w_eval(f, i):
token = f.stack.pop()
f.execute_token(token)
return i+1
def w_no_op(f, i):
return i+1
def w_forth(f, i):
f.stack.push(f)
return i+1
def w_current_ns(f, i):
f.stack.push(f.namespace)
return i + 1
def w_ns(f, i):
name = f.stack.pop()
if name in f.namespaces:
f.namespace = f.namespaces[name]
else:
new_ns = f.make_namespace(name, {}, [f.forth_ns])
f.namespace = new_ns
return i + 1
def w_alias(f, i):
new_name = f.stack.pop()
old_name = f.stack.pop()
f.namespace[new_name] = f.namespace[old_name]
return i + 1
def w_require(f, i):
name = f.stack.pop()
m = importlib.import_module(name)
import_native_module(f, m, name)
return i + 1
def source(f, path):
old_source_f = f.namespace.get('*source*', None)
old_namespace = f.namespace
try:
f.execute_file(path)
finally:
f.namespace['*source*'] = old_source_f
f.namespace = old_namespace
def w_load(f, i):
path = f.stack.pop()
source(f, path)
return i + 1
def w_source(f, i):
path = f.stack.pop()
if os.path.isabs(path):
source(f, path)
return i+1
relative_dir = os.path.dirname(f.evaluate_string('*source*'))
relative_path = f'{relative_dir}/{path}'
if os.path.exists(relative_path):
source(f, relative_path)
return i+1
source(f, path)
return i+1
def execute_f(name, instructions):
#print('execute_f:', name, len(instructions))
def inner(forth, i, debug=False):
#print('inner f:', name)
#print('inner f:', len(instructions))
j = 0
while j >= 0:
#print(j, '=>', instructions[j])
new_j = instructions[j](forth, j)
if new_j == None:
print(f'Instruction {instructions[j]} None')
raise RuntimeError
j = new_j
return i + 1
return inner
def ifnot_jump_f(n):
def ifnot_jump(forth, i):
x = forth.stack.pop()
if not x:
return i+n
return i+1
return ifnot_jump
def jump_f(n):
def do_jump(forth, i):
return n+i
return do_jump
def w_recur(f, i):
return 0
def w_import(f, i):
name = f.stack.pop()
m = importlib.import_module(name)
f.namespace[name] = const_f(m)
return i+1
def w_call(f, i):
func = f.stack.pop()
args = f.stack.pop()
# print('f', f, 'args', args)
try:
result = func(*args)
except:
print(f'Error executing {func}{list(args)}')
raise
# print('result', result)
f.stack.push(result)
return i+1
def w_nl(f, i):
print()
return i+1
def w_return(f, i):
return -9999;
def w_colon(f, i):
f.compiler = Compiler()
def w_semi(forth, i):
forth.compiler.add_instruction(w_return)
name = forth.compiler.name
word_f = execute_f(name, forth.compiler.instructions)
forth.namespace[name] = word_f
forth.compiler = None
return i+1
w_semi.__dict__['immediate'] = True
def w_should_not_happen(forth, i):
print('Should not execute this word!')
raise ValueError
def w_if(forth, i):
#print('w_if')
compiler = forth.compiler
compiler.push_offset()
compiler.push_offset()
compiler.add_instruction(w_should_not_happen)
return i+1
w_if.__dict__['immediate'] = True
def w_then(forth, i):
compiler = forth.compiler
else_offset = compiler.pop_offset()
if_offset = compiler.pop_offset()
then_offset = compiler.offset()
if else_offset == if_offset:
delta = then_offset - if_offset
compiler.instructions[if_offset] = ifnot_jump_f(delta)
else:
if_delta = else_offset - if_offset + 1
compiler.instructions[if_offset] = ifnot_jump_f(if_delta)
else_delta = then_offset - else_offset
compiler.instructions[else_offset] = jump_f(else_delta)
return i+1
w_then.__dict__['immediate'] = True
def w_else(forth, i):
compiler = forth.compiler
compiler.pop_offset()
compiler.push_offset()
compiler.add_instruction(w_should_not_happen)
return i+1
w_else.__dict__['immediate'] = True
def w_do(forth, i):
#print('w_do')
compiler = forth.compiler
compiler.push_offset()
compiler.add_instruction(w_should_not_happen)
return i+1
w_do.__dict__['immediate'] = True
def w_while(forth, i):
compiler = forth.compiler
do_offset = compiler.pop_offset()
while_offset = compiler.offset()
delta = do_offset - while_offset
compiler.instructions[if_offset] = ifnot_jump_f(delta)
return i+1
w_while.__dict__['immediate'] = True
def w_begin(forth, i):
compiler = forth.compiler
compiler.push_offset()
return i+1
w_begin.__dict__['immediate'] = True
def w_until(forth, i):
compiler = forth.compiler
begin_offset = compiler.pop_offset()
until_offset = compiler.offset()
delta = begin_offset - until_offset
#print('Delta:', delta)
compiler.instructions.append(ifnot_jump_f(delta))
return i+1
w_until.__dict__['immediate'] = True
def w_dump(f, i):
f.dump()
return i+1
def w_idump(f, i):
f.dump()
return i+1
w_idump.__dict__['immediate'] = True
def w_stack(f, i):
print("Stack:", end=' ')
for x in f.stack:
print(f'{repr(x)}', end=' ')
print()
return i+1
def w_enlist(f, i):
# print("Enlist!")
x = f.stack.pop()
# print("Popped", x)
f.stack.push([x])
return i+1

157
sallyforth/data_words.py Normal file
View file

@ -0,0 +1,157 @@
from unique import Unique
def w_unique(f, ip): # pushes a uique object.
f.stack.push(Unique())
return ip+1
def w_map(f, ip):
word = f.stack.pop()
l = f.stack.pop()
word_f = f.namespace.get(word, None)
result = []
for item in l:
f.stack.push(item)
word_f(f, 0)
result.append(f.stack.pop())
f.stack.push(result)
return ip+1
def w_reduce(f, ip):
l = f.stack.pop()
word = f.stack.pop()
word_f = f.namespace.get(word, None)
if len(l) <= 0:
f.stack.push(None)
elif len(l) == 1:
f.stack.push(l[0])
else:
result = l[0]
l = l[1::-1]
for item in l:
f.stack.push(result)
f.stack.push(item)
word_f(f, 0)
result = f.stack.pop()
f.stack.push(result)
return ip+1
def w_bounded_list(f, ip):
"""Create a list from delimted values on the stack.
[list]
(marker a b c marker -- [a b c]
"""
marker = f.stack.pop()
l = []
if f.stack.empty():
raise ValueError("Stack underflow")
x = f.stack.pop()
while x != marker:
l.append(x)
if f.stack.empty():
raise ValueError("Stack underflow")
x = f.stack.pop()
l.reverse()
f.stack.push(l)
return ip+1
def w_to_arglist(f, ip):
l = f.stack.pop()
f.stack.push(Arglist(l))
return ip+1
def w_list(f, ip): # ->list
n = f.stack.pop()
l = []
for i in range(n):
l.append(f.stack.pop())
f.stack.push(l)
return ip+1
def w_thread(f, i): # @@
contents = f.stack.pop()
result = contents[0]
for field in contents[1::]:
if isinstance(field, str) and hasattr(result, field):
result = getattr(result, field) # result.field
elif isinstance(field, Arglist):
result = result(*field) # result(*field)
else:
result = result[field] # result[field]
f.stack.push(result)
return i+1
ListMarker = object()
def w_startlist(f, i): # [
f.stack.push(ListMarker)
return i+1
def w_endlist(f, i): # ]
l = []
x = f.stack.pop()
while x != ListMarker:
l.append(x)
x = f.stack.pop()
l.reverse()
f.stack.push(l)
return i+1
MapMarker = object()
def w_startmap(f, i): # {
f.stack.push(MapMarker)
return i+1
def w_endmap(f, ip): # }
l = []
x = f.stack.pop()
while x != MapMarker:
l.append(x)
x = f.stack.pop()
if (len(l) % 2) != 0:
print('Maps need even number of entries.')
return i+1
l.reverse()
result = {}
for i in range(0, len(l), 2):
result[l[i]] = l[i+1]
f.stack.push(result)
return ip+1
def w_list_to_map(f, ip): # list->map
l = f.stack.pop()
result = {}
for i in range(0, len(l), 2):
result[l[i]] = l[i+1]
f.stack.push(result)
return ip+1
def w_get(f, i):
name = f.stack.pop()
m = f.stack.pop()
result = m[name]
f.stack.push(result)
return i+1
def w_getattribute(f, i):
name = f.stack.pop()
x = f.stack.pop()
result = x.__getattribute__(name)
f.stack.push(result)
return i+1
def w_def(f, i):
value = f.stack.pop()
name = f.stack.pop()
f.defvar(name, value)
# print('name', name, 'value', value)
return i+1

47
sallyforth/init.sf Normal file
View file

@ -0,0 +1,47 @@
'*prompt* "sallySh> " def
: child-run-prog (argv pid -- <<exit>>)
drop \ Get rid of 0 pid.
execvp \ Run the program
;
: parent-wait (cmd pid -- exit-status)
"parent" p
waitpid
"Child status:" p p
drop
;
: run-prog (argv -- status)
'child-run-prog 'parent-wait fork
;
'cmd-marker unique def
: a cmd-marker ;
: z
cmd-marker bounded_list
stack
run-prog
;
'os import
: expandvars #os.path.expandvars !!1 ;
: expanduser #os.path.expanduser !!1 ;
: expand expanduser expandvars ;
\: prompt-and-run ( -- prog-status)
\ ">> " read-line
\ dup "x" =
\ if
\ "Exit!" p
\ drop
\ else
\ tokenize
\ 'expand map
\ run-prog
\ recur
\ then
\;

12
sallyforth/io.sf Normal file
View file

@ -0,0 +1,12 @@
: open builtins.open !!1 ;
: close <. $? 'close .> !!0 drop ;
: read-file (path -- contents) open dup <. $? 'read .> !!0 swap close ;
: read-lines (path -- contents) open dup <. $? 'readlines .> !!0 swap close ;
: read-line (prompt -- input-line)
builtins.input !!1
;
: read-next nexttoken second ;

View file

@ -1,8 +1,9 @@
import sys
from os import path
from words import *
import words
from lex import is_string, Tokenizer
import basic_words, data_words, operator_words, stack_words, os_words
from basic_words import const_f, w_enlist
#from lex import is_string, Tokenizer
import tokenstream as ts
from stack import Stack
from namespace import Namespace
@ -17,7 +18,7 @@ def to_number(token):
class Forth:
def __init__(self, startup=None):
self.tokenizer = Tokenizer(self)
self.streams = Stack()
self.stack = Stack()
self.namespaces = {}
initial_defs = {
@ -34,7 +35,11 @@ class Forth:
self.forth_ns = self.make_namespace('forth', initial_defs)
user_ns = self.make_namespace('user', {}, [self.forth_ns])
self.forth_ns.import_from_module(words, 'w_')
self.forth_ns.import_from_module(basic_words, 'w_')
self.forth_ns.import_from_module(data_words, 'w_')
self.forth_ns.import_from_module(operator_words, 'w_')
self.forth_ns.import_from_module(stack_words, 'w_')
self.forth_ns.import_from_module(os_words, 'w_')
self.namespace = self.forth_ns
self.compiler = None
@ -49,55 +54,120 @@ class Forth:
def defvar(self, name, value):
self.namespace[name] = const_f(value)
def py_evaluate(self, token, *args):
#print(f'Evaluate: token [{token}] args <<{args}>>')
def compiling(self):
return self.compiler
def _compile_token(self, kind, token):
print(f"compile: {self.compiler.name}: {kind} {token}")
if self.compiler.name == None:
self.compiler.name = token
return
if kind in ['dqstring', 'sqstring']:
self.compiler.add_instruction(const_f(token))
return
if token in self.namespace:
word = self.namespace[token]
if 'immediate' in word.__dict__:
word(self, 0)
else:
self.compiler.add_instruction(self.namespace[token])
return
n = to_number(token)
if n == None:
print(f'{token}? Compile of {self.compiler.name} terminated.')
self.compiler = None
else:
self.compiler.add_instruction(const_f(n))
def _eval_token(self, kind, token):
if kind in ['dqstring', 'sqstring']:
self.stack.push(token)
return
if token in self.namespace:
# print("executing ", token)
self.namespace[token](self, 0)
return
n = to_number(token)
if n == None:
print(f'{token}?')
else:
self.stack.push(n)
def execute_token(self, kind, token):
#print(f'execute kind {kind} token: {token}')
kts = self.macro_expand_token(kind, token)
#print(kts)
for kt in kts:
this_kind, this_token = kt
#print(f'execute this {this_kind} {this_token}')
if not self.compiling():
self._eval_token(this_kind, this_token)
else:
self._compile_token(this_kind, this_token)
def XX_execute_token_stream(self, s):
kind, token = s.get_token()
while kind != 'eof':
self.execute_token(kind, token)
kind, token = s.get_token()
def execute_current_stream(self):
s = self.streams.peek()
print("exec current s:", s)
kind, token = s.get_token()
while kind != 'eof':
self.execute_token(kind, token)
kind, token = s.get_token()
self.streams.pop()
def execute_token_stream(self, s):
print("exec token stream:", s)
self.streams.push(s)
self.execute_current_stream()
def execute_string(self, s):
token_stream = ts.string_token_stream(s)
return self.execute_token_stream(token_stream)
def evaluate_string(self, s):
self.execute_string(s)
return self.stack.pop()
def read_next_token(self):
s = self.streams.peek()
return s.get_token()
def py_evaluate(self, s, *args):
print(f'Evaluate: token [{token}] args <<{args}>>')
rargs = list(args)
rargs.reverse()
if rargs:
for a in rargs:
# print("pushing", a);
self.stack.push(a)
#print(f'Before eval stack is {str(self.stack)}')
return self.evaluate_token(token)
print(f'Before eval stack is {str(self.stack)}')
return self.evaluate_string(s)
def evaluate_token(self, token):
#print("evaluate token: ", token)
self.execute_token(token)
return self.stack.pop()
def compiling(self):
return self.compiler
def execute_line(self, line):
tokens = self.tokenizer.tokenize(line)
self.execute_tokens(tokens)
def execute_tokens(self, tokens):
for token in tokens:
# print("token:", token)
if not self.compiling():
self.execute_token(token)
else:
self.compile_token(token)
def macro_expand_token(self, token):
def macro_expand_token(self, kind, token):
if len(token) <= 0 or token[0] != '#':
return [token]
return [[kind, token]]
tag = token[1:]
parts = tag.split('.')
result = [ '<.', parts[0] ]
result = [['word', '<.'], ['word', parts[0]]]
for part in parts[1::]:
result.append("'" + part)
result.append('.>')
result.append(['sqstring', part])
result.append(['word', '.>'])
print(result)
return result
def macro_expand_tokens(self, tokens):
results = []
for token in tokens:
results.extend(self.macro_expand_token(token))
return results
def set_ns(self, ns_name):
if ns_name in self.namespaces:
self.namespace = self.namespaces[ns_name]
@ -116,54 +186,11 @@ class Forth:
old_namespace = self.namespace
self.defvar('*source*', fpath)
with open(fpath) as f:
line = f.readline()
while line:
self.execute_line(line)
line = f.readline()
fts = ts.file_token_stream(f)
self.execute_token_stream(fts)
self.namespace['*source*'] = old_source
self.namespace = old_namespace
def compile_token(self, token):
if self.compiler.name == None:
self.compiler.name = token
return
if is_string(token):
self.compiler.add_instruction(const_f(token[1::]))
return
if token in self.namespace:
word = self.namespace[token]
if 'immediate' in word.__dict__:
word(self, 0)
else:
self.compiler.add_instruction(self.namespace[token])
return
n = to_number(token)
if n == None:
print(f'{token}? Compile of {self.compiler.name} terminated.')
self.compiler = None
else:
self.compiler.add_instruction(const_f(n))
def execute_token(self, token):
# print("x token:", token)
if is_string(token):
self.stack.push(token[1::])
return
if token in self.namespace:
# print("executing ", token)
self.namespace[token](self, 0)
return
n = to_number(token)
if n == None:
print(f'{token}?')
else:
self.stack.push(n)
def dump(self):
print('Forth:', self)
print('Stack:', self.stack)

View file

@ -1,7 +1,3 @@
import sys
import readline
from os import path
def is_string(token):
return token[0] == '"' or token[0] == "'"

View file

@ -20,7 +20,7 @@
[x] \ Do a[0..n].
;
: drop (n list -- all-but-first-n-items)
: skip (n list -- all-but-first-n-items)
swap nil slice \ Make the n..None slice.
[x]
;
@ -29,7 +29,11 @@
1 ->list *
;
: rest (list -- all-but-first) 1 swap drop ;
: len builtins.len !!1 ;
: empty? len zero? ;
: rest (list -- all-but-first) 1 swap skip ;
: rrest (list -- rest-of-rest) rest rest ;
: rrrest (list -- all-but-first) rest rest rest ;

View file

@ -38,7 +38,9 @@ class Namespace:
return self[key]
def __contains__(self, key):
#print(f'Namespace contains {key}')
if self.contents.__contains__(key):
#print(self.contents[key])
return True
for r in self.refers:
if r.__contains__(key):

View file

@ -0,0 +1,67 @@
def w_gt(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b > a)
return i+1
def w_lt(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b < a)
return i+1
def w_eq(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(a==b)
return i+1
def w_le(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b<=a)
return i+1
def w_ge(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b>=a)
return i+1
def w_add(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b+a)
return i+1
def w_mul(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b*a)
return i+1
def w_sub(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b-a)
return i+1
def w_div(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b/a)
return i+1
def w_and(f, i):
f.stack.push(f.stack.pop() and f.stack.pop())
return i+1
def w_or(f, i):
f.stack.push(f.stack.pop() or f.stack.pop())
return i+1
def w_not(f, i):
f.stack.push(not f.stack.pop())
return i+1

41
sallyforth/os_words.py Normal file
View file

@ -0,0 +1,41 @@
import os
import sys
def w_fork(f, i):
parent_word = f.stack.pop()
child_word = f.stack.pop()
parent_f = f.namespace.get(parent_word, None)
child_f = f.namespace.get(child_word, None)
pid = os.fork()
f.stack.push(pid)
if pid == 0:
print("child:", pid)
child_f(f, 0)
else:
print("parent:", pid)
parent_f(f, 0)
return i+1
def w_execvp(f, i):
args = f.stack.pop()
path = args[0]
print(f"path {path} args: {args}")
os.execvp(path, args)
return i+1
def w_waitpid(f, i):
pid = f.stack.pop()
result = os.waitpid(pid, 0)
f.stack.push(result)
return i+1
def w_exit(f, i):
n = f.stack.pop()
sys.exit(n)
return i+1
def w_exit_bang(f, i):
n = f.stack.pop()
os._exit(n)
return i+1

View file

@ -1,9 +1,10 @@
import os
import sys
import atexit
from kernel import Forth
import readline
import traceback
from kernel import Forth
from tokenstream import prompt_token_stream
HistoryFile=".sallyforth"
@ -30,6 +31,9 @@ def setup_readline(history_path, f):
def save_history():
readline.write_history_file(history_path)
atexit.register(save_history)
def prompt_f():
return f.evaluate_string('*prompt*')
return prompt_token_stream(prompt_f)
def setup_forth():
source_dir = os.path.dirname(os.path.abspath(__file__))
@ -39,34 +43,13 @@ def setup_forth():
f = Forth(startup_file)
else:
f = Forth()
return f
def repl(f):
while True:
p = f.evaluate_token('*prompt*')
try:
line = input(p)
except KeyboardInterrupt:
print("<<interrupt>>")
f.stack.reset()
line = ''
except EOFError:
break
try:
f.execute_line(line)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
print("Error:", exc_type)
print("Error:", exc_value)
print("Error:", exc_traceback)
traceback.print_tb(exc_traceback)
def repl(stream, f):
f.execute_token_stream(stream)
if __name__ == "__main__":
f = setup_forth()
setup_readline(hist_file, f)
repl(f)
stream = setup_readline(hist_file, f)
repl(stream, f)
print("Bye!")

View file

@ -4,14 +4,14 @@ class Stack:
self.stack = 100 * [None]
def push(self, x):
# print("stack push", x)
#print("stack push", x)
self.top += 1
self.stack[self.top] = x
return x
def pop(self):
result = self.stack[self.top]
# print("stack pop", result)
#print("stack pop", result)
self.top -= 1
if self.top < -1:
print("stack overpop")

103
sallyforth/stack_words.py Normal file
View file

@ -0,0 +1,103 @@
def w_px(f, i):
args = f.stack.pop()
name = f.stack.pop()
m = f.stack.pop()
func = m.__dict__[name]
result = func(*args)
f.stack.push(result)
return i+1
def w_reset(f, i):
a = f.stack.reset()
return i+1
def w_dot(f, i):
a = f.stack.pop()
print(a, end='')
return i+1
def w_splat(f, i):
l = f.stack.pop()
l.reverse()
for x in l:
f.stack.push(x)
return i+1
def w_dup(f, i):
x = f.stack.peek()
f.stack.push(x)
return i+1
def w_tmb(f, i): # A noop
# t = f.stack.pop()
# m = f.stack.pop()
# b = f.stack.pop()
# f.stack.push(b)
# f.stack.push(m)
# f.stack.push(t)
return i+1
def w_tbm(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(m)
f.stack.push(b)
f.stack.push(t)
return i+1
def w_bmt(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(t)
f.stack.push(m)
f.stack.push(b)
return i+1
def w_btm(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(m)
f.stack.push(t)
f.stack.push(b)
return i+1
def w_mtb(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(b)
f.stack.push(t)
f.stack.push(m)
return i+1
def w_mbt(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(t)
f.stack.push(b)
f.stack.push(m)
return i+1
def w_rot(f, i):
c = f.stack.pop()
b = f.stack.pop()
a = f.stack.pop()
f.stack.push(b)
f.stack.push(c)
f.stack.push(a)
return i+1
def w_drop(f, i):
f.stack.pop()
return i+1
def w_swap(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(a)
f.stack.push(b)
return i+1

View file

@ -18,6 +18,7 @@
'colon ': alias
'semi '; alias
'thread '@@ alias
'exit_bang 'exit! alias
'bounded_list '[list] alias
'list '->list alias
'to_arglist '->arglist alias
@ -47,6 +48,7 @@
\ Look up attributes on a value.
: <. [ ;
: .> ] thread ;
: $? swap ;
\ Call native functions with various # arguments.
: !!0 [] swap !! ;
@ -60,6 +62,7 @@
: {} ( -- <empty map>) { } ;
\ Make a set.
'set-marker unique def
@ -87,6 +90,8 @@
: hello "Hello" . nl ;
: >0 0 > ;
: =0 0 = ;
: <1 1 < ;
: <0 0 < ;
: >1 1 > ;
: <1 1 < ;
@ -118,7 +123,10 @@
: .!!2 (obj a1 a2 method-name -- result ) swap 2 ->list swap .!! ;
: .!!3 (obj a1 a2 a3 method-name -- result ) swap 3 ->list swap .!! ;
\ todo : tokenize #forth.tokenizer.tokenize !!1 ;
"string.sf" source
"list.sf" source
"io.sf" source
"init.sf" source-if-exists

87
sallyforth/tokenstream.py Normal file
View file

@ -0,0 +1,87 @@
import io
class PromptInputStream:
def __init__(self, prompt_f):
self.prompt_f = prompt_f
self.buffer = []
def getc(self):
try:
if len(self.buffer) == 0:
prompt = self.prompt_f()
line = input(prompt)
line += '\n'
self.buffer = list(line)
self.buffer.reverse()
return self.buffer.pop()
except EOFError:
return ''
class TokenStream:
def __init__(self, read_f):
self.read_f = read_f
def whitespace(self, ch):
return ch in [' ', '\t', '\n']
def get_token(self):
state = 'start'
token = ''
while True:
ch = self.read_f()
#print(f'ch: {ch} typech {type(ch)} state {state}')
if ch in ['', None]:
if state in ['word', 'sqstring', 'dqstring']:
return [state, token]
return ['eof', '']
elif state == 'start' and ch == '\\':
state = 'lcomment'
elif state == 'lcomment' and ch == '\n':
state = 'start'
elif state == 'start' and ch == '(':
state = 'icomment'
elif state == 'icomment' and ch == ')':
state = 'start'
elif state == 'start' and self.whitespace(ch):
continue
elif state == 'start' and ch == '"':
state = 'dqstring'
elif state == 'dqstring' and ch == '"':
return [state, token]
elif state == 'start' and ch == "'":
state = 'sqstring'
elif state == 'start':
state = 'word'
token += ch
elif state in ['word', 'sqstring'] and \
self.whitespace(ch):
return state, token
elif state in ['word', 'dqstring', 'sqstring']:
token += ch
def file_token_stream(f):
return TokenStream(lambda : f.read(1))
def string_token_stream(s):
sio = io.StringIO(s)
return file_token_stream(sio)
def prompt_token_stream(prompt_f):
pis = PromptInputStream(prompt_f)
return TokenStream(pis.getc)
if __name__ == "__main__":
x = 0
def pmt():
global x
x += 1
return f'Yes{x}>> '
pis = PromptInputStream(pmt)
ts = TokenStream(pis.getc)
kind, token = ts.get_token()
while kind != 'eof':
print(kind, token)
kind, token = ts.get_token()

3
sallyforth/unique.py Normal file
View file

@ -0,0 +1,3 @@
class Unique:
def __str__(self):
return f'Unique[{id(self)}]'

View file

@ -1,583 +0,0 @@
from inspect import isfunction, isbuiltin
import importlib
import os
from compiler import Compiler
from arglist import Arglist
class Unique:
def __str__(self):
return f'Unique[{id(self)}]'
def const_f(value):
def x(f, i):
f.stack.push(value)
return i + 1
return x
def native_function_handler(func):
def handle(forth, i):
args = forth.stack.pop()
#print(f"Native fun, calling {func}({args})")
result = func(*args)
#print(f'Result: {result}')
forth.stack.push(result)
#print("pushed result")
return i + 1
return handle
def import_native_module(forth, m, alias=None, excludes=[]):
if not alias:
alias = m.__name__
raw_names = dir(m)
names = [x for x in raw_names if x not in excludes]
for name in names:
localname = f'{alias}.{name}'
val = m.__getattribute__(name)
forth.namespace[localname] = const_f(val)
def w_eval(f, i):
token = f.stack.pop()
f.execute_token(token)
return i+1
def w_no_op(f, i):
return i+1
def w_enlist(f, i):
# print("Enlist!")
x = f.stack.pop()
# print("Popped", x)
f.stack.push([x])
return i+1
def w_forth(f, i):
f.stack.push(f)
return i+1
def w_current_ns(f, i):
f.stack.push(f.namespace)
return i + 1
def w_ns(f, i):
name = f.stack.pop()
if name in f.namespaces:
f.namespace = f.namespaces[name]
else:
new_ns = f.make_namespace(name, {}, [f.forth_ns])
f.namespace = new_ns
return i + 1
def w_alias(f, i):
new_name = f.stack.pop()
old_name = f.stack.pop()
f.namespace[new_name] = f.namespace[old_name]
return i + 1
def w_require(f, i):
name = f.stack.pop()
m = importlib.import_module(name)
import_native_module(f, m, name)
return i + 1
def source(f, path):
old_source_f = f.namespace.get('*source*', None)
old_namespace = f.namespace
try:
f.execute_file(path)
finally:
f.namespace['*source*'] = old_source_f
f.namespace = old_namespace
def w_load(f, i):
path = f.stack.pop()
source(f, path)
return i + 1
def w_source(f, i):
path = f.stack.pop()
if os.path.isabs(path):
source(f, path)
return i+1
relative_dir = os.path.dirname(f.evaluate_token('*source*'))
relative_path = f'{relative_dir}/{path}'
if os.path.exists(relative_path):
source(f, relative_path)
return i+1
source(f, path)
return i+1
def execute_f(name, instructions):
#print('execute_f:', name, len(instructions))
def inner(forth, i, debug=False):
#print('inner f:', name)
#print('inner f:', len(instructions))
j = 0
while j >= 0:
#print(j, '=>', instructions[j])
j = instructions[j](forth, j)
return i + 1
return inner
def ifnot_jump_f(n):
def ifnot_jump(forth, i):
# print('If not jump:')
x = forth.stack.pop()
# print('==>value:', x)
if not x:
# print('==>', x, ' is false')
# print('==>returning', n)
return i+n
# print('==>returning 1')
return i+1
return ifnot_jump
def jump_f(n):
def do_jump(forth, i):
return n+i
return do_jump
def w_recur(f, i):
return 0
def w_import(f, i):
name = f.stack.pop()
m = importlib.import_module(name)
f.namespace[name] = const_f(m)
return i+1
def w_call(f, i):
func = f.stack.pop()
args = f.stack.pop()
# print('f', f, 'args', args)
result = func(*args)
# print('result', result)
f.stack.push(result)
return i+1
def w_px(f, i):
args = f.stack.pop()
name = f.stack.pop()
m = f.stack.pop()
func = m.__dict__[name]
result = func(*args)
f.stack.push(result)
return i+1
def w_unique(f, ip): # pushes a uique object.
f.stack.push(Unique())
return ip+1
def w_map(f, ip):
l = f.stack.pop()
word = f.stack.pop()
word_f = f.namespace.get(word, None)
result = []
for item in l:
f.stack.push(item)
word_f(f, 0)
result.append(f.stack.pop())
f.stack.push(result)
return ip+1
def w_reduce(f, ip):
l = f.stack.pop()
word = f.stack.pop()
word_f = f.namespace.get(word, None)
if len(l) <= 0:
f.stack.push(None)
elif len(l) == 1:
f.stack.push(l[0])
else:
result = l[0]
l = l[1::-1]
for item in l:
f.stack.push(result)
f.stack.push(item)
word_f(f, 0)
result = f.stack.pop()
f.stack.push(result)
return ip+1
def w_bounded_list(f, ip):
"""Create a list from delimted values on the stack.
[list]
(marker a b c marker -- [a b c]
"""
marker = f.stack.pop()
l = []
x = f.stack.pop()
while x != marker:
l.append(x)
x = f.stack.pop()
l.reverse()
f.stack.push(l)
return ip+1
def w_to_arglist(f, ip):
l = f.stack.pop()
f.stack.push(Arglist(l))
return ip+1
def w_list(f, ip): # ->list
n = f.stack.pop()
l = []
for i in range(n):
l.append(f.stack.pop())
f.stack.push(l)
return ip+1
def w_thread(f, i): # @@
contents = f.stack.pop()
result = contents[0]
for field in contents[1::]:
if isinstance(field, str) and hasattr(result, field):
result = getattr(result, field) # result.field
elif isinstance(field, Arglist):
result = result(*field) # result(*field)
else:
result = result[field] # result[field]
f.stack.push(result)
return i+1
ListMarker = object()
def w_startlist(f, i): # [
f.stack.push(ListMarker)
return i+1
def w_endlist(f, i): # ]
l = []
x = f.stack.pop()
while x != ListMarker:
l.append(x)
x = f.stack.pop()
l.reverse()
f.stack.push(l)
return i+1
MapMarker = object()
def w_startmap(f, i): # {
f.stack.push(MapMarker)
return i+1
def w_endmap(f, ip): # }
l = []
x = f.stack.pop()
while x != MapMarker:
l.append(x)
x = f.stack.pop()
if (len(l) % 2) != 0:
print('Maps need even number of entries.')
return i+1
l.reverse()
result = {}
for i in range(0, len(l), 2):
result[l[i]] = l[i+1]
f.stack.push(result)
return ip+1
def w_list_to_map(f, ip): # list->map
l = f.stack.pop()
result = {}
for i in range(0, len(l), 2):
result[l[i]] = l[i+1]
f.stack.push(result)
return ip+1
def w_get(f, i):
name = f.stack.pop()
m = f.stack.pop()
result = m[name]
f.stack.push(result)
return i+1
def w_getattribute(f, i):
name = f.stack.pop()
x = f.stack.pop()
result = x.__getattribute__(name)
f.stack.push(result)
return i+1
def w_def(f, i):
value = f.stack.pop()
name = f.stack.pop()
f.defvar(name, value)
# print('name', name, 'value', value)
return i+1
def w_gt(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b > a)
return i+1
def w_lt(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b < a)
return i+1
def w_eq(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(a==b)
return i+1
def w_le(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b<=a)
return i+1
def w_ge(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b>=a)
return i+1
def w_add(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b+a)
return i+1
def w_mul(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b*a)
return i+1
def w_sub(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b-a)
return i+1
def w_div(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(b/a)
return i+1
def w_reset(f, i):
a = f.stack.reset()
return i+1
def w_dot(f, i):
a = f.stack.pop()
print(a, end='')
return i+1
def w_splat(f, i):
l = f.stack.pop()
l.reverse()
for x in l:
f.stack.push(x)
return i+1
def w_dup(f, i):
x = f.stack.peek()
f.stack.push(x)
return i+1
def w_tmb(f, i): # A noop
# t = f.stack.pop()
# m = f.stack.pop()
# b = f.stack.pop()
# f.stack.push(b)
# f.stack.push(m)
# f.stack.push(t)
return i+1
def w_tbm(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(m)
f.stack.push(b)
f.stack.push(t)
return i+1
def w_bmt(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(t)
f.stack.push(m)
f.stack.push(b)
return i+1
def w_btm(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(m)
f.stack.push(t)
f.stack.push(b)
return i+1
def w_mtb(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(b)
f.stack.push(t)
f.stack.push(m)
return i+1
def w_mbt(f, i):
t = f.stack.pop()
m = f.stack.pop()
b = f.stack.pop()
f.stack.push(t)
f.stack.push(b)
f.stack.push(m)
return i+1
def w_rot(f, i):
c = f.stack.pop()
b = f.stack.pop()
a = f.stack.pop()
f.stack.push(b)
f.stack.push(c)
f.stack.push(a)
return i+1
def w_drop(f, i):
f.stack.pop()
return i+1
def w_swap(f, i):
a = f.stack.pop()
b = f.stack.pop()
f.stack.push(a)
f.stack.push(b)
return i+1
def w_nl(f, i):
print()
return i+1
def w_return(f, i):
return -9999;
def w_colon(f, i):
f.compiler = Compiler()
def w_semi(forth, i):
forth.compiler.add_instruction(w_return)
name = forth.compiler.name
word_f = execute_f(name, forth.compiler.instructions)
forth.namespace[name] = word_f
forth.compiler = None
return i+1
w_semi.__dict__['immediate'] = True
def w_should_not_happen(forth, i):
print('Should not execute this word!')
raise ValueError
def w_if(forth, i):
#print('w_if')
compiler = forth.compiler
compiler.push_offset()
compiler.push_offset()
compiler.add_instruction(w_should_not_happen)
return i+1
w_if.__dict__['immediate'] = True
def w_then(forth, i):
compiler = forth.compiler
else_offset = compiler.pop_offset()
if_offset = compiler.pop_offset()
then_offset = compiler.offset()
if else_offset == if_offset:
delta = then_offset - if_offset
compiler.instructions[if_offset] = ifnot_jump_f(delta)
else:
if_delta = else_offset - if_offset + 1
compiler.instructions[if_offset] = ifnot_jump_f(if_delta)
else_delta = then_offset - else_offset
compiler.instructions[else_offset] = jump_f(else_delta)
return i+1
w_then.__dict__['immediate'] = True
def w_else(forth, i):
compiler = forth.compiler
compiler.pop_offset()
compiler.push_offset()
compiler.add_instruction(w_should_not_happen)
return i+1
w_else.__dict__['immediate'] = True
def w_do(forth, i):
#print('w_do')
compiler = forth.compiler
compiler.push_offset()
compiler.add_instruction(w_should_not_happen)
return i+1
w_do.__dict__['immediate'] = True
def w_while(forth, i):
compiler = forth.compiler
do_offset = compiler.pop_offset()
while_offset = compiler.offset()
delta = do_offset - while_offset
compiler.instructions[if_offset] = ifnot_jump_f(delta)
return i+1
w_while.__dict__['immediate'] = True
def w_begin(forth, i):
compiler = forth.compiler
compiler.push_offset()
return i+1
w_begin.__dict__['immediate'] = True
def w_until(forth, i):
compiler = forth.compiler
begin_offset = compiler.pop_offset()
until_offset = compiler.offset()
delta = begin_offset - until_offset
#print('Delta:', delta)
compiler.instructions.append(ifnot_jump_f(delta))
return i+1
w_until.__dict__['immediate'] = True
def w_dump(f, i):
f.dump()
return i+1
def w_idump(f, i):
f.dump()
return i+1
w_idump.__dict__['immediate'] = True
def w_stack(f, i):
print("Stack:", end=' ')
for x in f.stack:
print(f'{repr(x)}', end=' ')
print()
return i+1