Adding reader macros. Turned tokenizer into a class, added list and string word.

This commit is contained in:
Russ Olsen 2020-04-22 08:57:14 -04:00
parent f10edf9fd7
commit f1bdf52b2b
8 changed files with 140 additions and 63 deletions

View file

@ -1,4 +1,3 @@
from lex import forth_prompt
from stack import Stack
class Compiler:

View file

@ -2,7 +2,7 @@ import sys
from os import path
from words import *
import words
from lex import forth_prompt, read_tokens, is_string, tokenize
from lex import is_string, Tokenizer
from stack import Stack
from namespace import Namespace
@ -17,10 +17,12 @@ def to_number(token):
class Forth:
def __init__(self, startup=None):
self.tokenizer = Tokenizer(self)
self.stack = Stack()
self.namespaces = {}
initial_defs = {
'*prompt*': const_f('SallyForth>> '),
'macroexpand': w_enlist,
'true': const_f(True),
'false': const_f(False),
'nil': const_f(None),
@ -46,15 +48,27 @@ class Forth:
def defvar(self, name, value):
self.namespace[name] = const_f(value)
def py_evaluate(self, token, *args):
#print(f'Evaluate: token [{token}] args <<{args}>>')
rargs = list(args)
rargs.reverse()
if rargs:
for a in rargs:
# print("pushing", a);
self.stack.push(a)
#print(f'Before eval stack is {str(self.stack)}')
return self.evaluate_token(token)
def evaluate_token(self, token):
#print("evaluate token: ", token)
self.execute_token(token)
return self.stack.pop()
def compiling(self):
return self.compiler
def execute_line(self, readline_f=forth_prompt):
tokens = read_tokens(readline_f)
def execute_line(self, line):
tokens = self.tokenizer.tokenize(line)
self.execute_tokens(tokens)
def execute_tokens(self, tokens):
@ -65,6 +79,20 @@ class Forth:
else:
self.compile_token(token)
def macro_expand_token(self, token):
if len(token) <= 0:
return [token]
if token[0] != '#':
return [token]
tag = token[1:]
return self.py_evaluate('macroexpand', tag)
def macro_expand_tokens(self, tokens):
results = []
for token in tokens:
results.extend(self.macro_expand_token(token))
return results
def set_ns(self, ns_name):
if ns_name in self.namespaces:
self.namespace = self.namespaces[ns_name]
@ -85,8 +113,7 @@ class Forth:
with open(fpath) as f:
line = f.readline()
while line:
tokens = tokenize(line)
self.execute_tokens(tokens)
self.execute_line(line)
line = f.readline()
self.namespace['*source*'] = old_source
self.namespace = old_namespace

View file

@ -8,7 +8,16 @@ def is_string(token):
def is_space(ch):
return ch == ' ' or ch == '\t' or ch == '\n'
def tokenize(s):
class Tokenizer:
def __init__(self, forth):
self.forth = forth
print("Tokenizer:", self.forth)
def tokenize(self, s):
raw_tokens = self.raw_tokenize(s)
return self.forth.macro_expand_tokens(raw_tokens)
def raw_tokenize(self, s):
state = 'start'
token = ''
tokens = []
@ -51,15 +60,3 @@ def tokenize(s):
if len(token) > 0:
tokens.append(token)
return tokens
def read_tokens(read_f):
line = read_f()
return tokenize(line)
def forth_prompt():
return input('SallyForth>> ')
def file_read_f(f):
def read_it():
return f.readline()
return read_it

36
sallyforth/list.sf Normal file
View file

@ -0,0 +1,36 @@
"List" p
: [n] (n list -- nth-item)
2 ->list \ Make the arg list.
<. builtins.list '__getitem__ .> \ Find the getitem method.
!! \ Call!
;
: first (list -- first-item) 0 swap [n] ;
: second (list -- second-item) 1 swap [n] ;
: third (list -- third-item) 2 swap [n] ;
: fourth (list -- fourth-item) 3 swap [n] ;
: last (list -- last-item) -1 swap [n] ;
: slice (start stop -- slice-obj)
swap
2 ->list
builtins.slice
!!
;
: take (n list -- first-n-items)
swap 0 swap slice \ Make the 0..n slice.
swap [n] \ Do a[0..n].
;
: drop (n list -- all-but-first-n-items)
swap nil slice \ Make the n..None slice.
swap [n]
;

View file

@ -2,7 +2,6 @@ import os
import sys
import atexit
from kernel import Forth
from lex import tokenize
import readline
import traceback
@ -55,9 +54,8 @@ def repl(f):
except EOFError:
break
tokens = tokenize(line)
try:
f.execute_tokens(tokens)
f.execute_line(line)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
print("Error:", exc_type)

View file

@ -101,6 +101,9 @@
: >>@ ] @@ ;
: >>! ] @@ [] swap !! ;
"string.sf" source
"list.sf" source
"init.sf" source-if-exists

7
sallyforth/string.sf Normal file
View file

@ -0,0 +1,7 @@
"String" p
: split (delimit str -- tokens) 2 ->list <. builtins.str 'split .> !! ;
: dot-split (str -- tokens) "." swap split ;

View file

@ -35,6 +35,16 @@ def import_native_module(forth, m, alias=None, excludes=[]):
else:
forth.namespace[localname] = const_f(val)
def w_no_op(f, i):
return i+1
def w_enlist(f, i):
print("Enlist!")
x = f.stack.pop()
print("Popped", x)
f.stack.push([x])
return i+1
def w_forth(f, i):
f.stack.push(f)
return i+1