So I am making a game where you type commands and move an npc,
from a simple idea where the commands were one letter long: f, b, l, r, (jump), a
to a system where you can put commands in functions using call expression like syntax, with arguments.
move(distance())
to, what I am currently transforming the system into, a full on (toy) programming language.
the tokenizer is done and ready for production.
but the parser doesn’t run, it gets 17 tokens to parse in the unit test, but it prints “hello?” and then “loopend once” and then nothing. no errors, just nothing.
here is the parser, beware of the hell, because I didn’t want to make a custom oop class for every statement type, so I made a token type, and then just used tables for statements:
local parser = {}
parser.__index = parser
--[[
EXPRESSIONS:
PRIMITIVE_EXPR - -1
BINAIRY - 0
UNARY - 1
TERNARY - 2
MEMBEREXPR - 4
CALLEXPR - 5
CASTEXPR - 6
ASSIGNMENTEXPR - 6.5
DECLARATIONS:
VARDEC - 7
FUNCDEC - 8
CLSDEC - 9
EVENTDEC - 10
ENUMDEC - 11
STATEMENTS:
IFSTMT - 12
WHILESTMT - 13
FORSTMT - 14
RETURNSTMT - 15
CONTINUESTMT - 16
BREAKSTMT - 17
TRY_CATCHSTMT - 18
SWITCHSTMT - 19
PRIMITIVES:
IDENTIFIER = 20
NONE = 21
BOOL = 22
NUMBER = 23
STRING = 24
]]
function parser.create(tkns: {any})
local new = setmetatable(parser, {})
new.result = {}
new.tokens = tkns
new.onCompleted = nil
return new
end
function parser:_advance()
table.remove(self.tokens, 1)
end
function parser:_cur()
return self.tokens[1]
end
function parser:_parse_literal(): {any}
local cur = self:_cur()
if cur:of(7) then
return {nodetype= 20, value = cur.value}
elseif cur:of(0) then
return {nodetype=21, value= "none"}
elseif cur:of(1) then
return {nodetype=24, value = cur.value}
elseif cur:of(3) then
return {nodetype=22, value= cur.value}
elseif cur:of(2) then
return {nodetype=23, value= cur.value}
elseif cur:of(6) then
return {nodetype=-100, value= ';'}
elseif cur:of(5) then
if cur.value ~= '-' then print("Invalid sequence of characters detected at: "..cur.line.." and column: "..cur.col) return {nodetype=21, "none"} end
self:_advance()--skip '-''
local unary = {nodetype=1, operand= self:_parse_assignexpr(), operator= '-'}
return unary
elseif cur:is(10, '(') then
local stmt = self:_parse_assignexpr()
assert(self:_cur():is(10, ')'),"Expected closing parenthesis at: "..cur.line.." at column: "..cur.col)
return stmt
else
warn(tostring(cur))
error("Unknown token detected at line: "..cur.line.." with column: "..cur.col)
return {nodetype=21, value="none"}
end
end
function parser:_parse_memberexpr(): {any}
local member = self:_parse_literal()
while self:_cur():is(5, '.') or self:_cur():is(5, '?') or self:_cur():is(8, '[') do
local op = self:_advance()
local iscomputed = false
local property = ""
if op:is(8, '[') then
property = self:_parse_assignexpr()
iscomputed = true
else
property = self:_parse_literal()
assert(property.nodetype == 20, "Expected identifier at line: "..op.line.." at column: "..op.col)
end
member = {nodetype= 4, reqproperty = property, is_computed = iscomputed, operator = op}
end
return member
end
function parser:_parse_getargs(): {any}
local args = {self:_parse_assignexpr()}
while self:_cur():is(11, ',') do
self:_advance()
table.insert(args, self:_parse_assignexpr())
end
return args
end
function parser:_parse_args(): {any}
local args = self:_cur():is(10, ')') and {} or self:_parse_getargs()
return args
end
function parser:_parse_call(obj: any): {any}
local expr = {invoked = obj, args = self:_parse_args(), nodetype=5}
if self:_cur():is(10, '(') then return self:_parse_call(expr) end
return expr
end
function parser:_parse_member_call(): {any}
local member = self:_parse_memberexpr()
if self:_cur():is(10, '(') then return self:_parse_call(member) end
return member
end
function parser:_parse_unary(): {any}
if self:_cur():is(4, "not") and not self:_cur():is(5, '++') and not self:_cur():is(5, '--') and not self:_cur():is(5, '//') and not self:_cur():is(5, '**') then
return self:_parse_member_call()
end
local op = self:_advance().value
return {nodetype=1, operand = self:_parse_logicalexpr(), operator = op}
end
function parser:_parser_multiplicative(): {any}
local left = self:_parse_unary()
while self:_cur():is(5, '*') or self:_cur():is(5, '/') or self:_cur():is(5, '%') or self:_cur():is(5, '^') or self:_cur():is(5, '~') do
local value = self:_advance().value
local right = self:_parse_unary()
left = {nodetype=0, left_operand = left, right_operand = right, operator = value}
end
return left
end
function parser:_parse_additive(): {any}
local left = self:_parse_multiplicative()
while self:_cur():is(5, '+') or self:_cur():is(5, '-') do
local value = self:_advance().value
local right = self:_parse_multiplicative()
left = {nodetype=0, left_operand = left, right_operand = right, operator = value}
end
return left
end
function parser:_parse_comparison(): {any}
local left = self:_parse_additive()
while self:_cur():is(5, '<') or self:_cur():is(5, '>') or self:_cur():is(5, '<=') or self:_cur():is(5, '>=') or self:_cur():is(5, '==') or self:_cur():is(5, '!=') do
local value = self:_advance().value
local right = self:_parse_additive()
left = {nodetype=0, left_operand = left, right_operand = right, operator = value}
end
return left
end
function parser:_parse_logicalexpr(): {any}
local left = self:_parse_comparison()
while self:_cur():is(4, "and") or self:_cur():is(4, "or") do
local value = self:_advance().value
local right = self:_parse_additive()
left = {nodetype=0, left_operand = left, right_operand = right, operator = value}
end
return left
end
function parser:_parse_assignexpr(): {any}
local left = self:_parse_logicalexpr()
if self:_cur():is(5, '=') then
self:_advance()
local right = self:_parse_assignexpr()
left = {nodetype = 6.5, assignee = left, value = right}
end
return left
end
function parser:_parse_var(): {any}
print("started var dec")
local isconst = self:_advance():is(4, "const")
local name = self:_advance()
assert(name:of(7), "Expected identifier for variable name at line: "..name.line.." at column: "..name.col)
print(tostring(name))--it is a token
if not self:_cur():is(5, '=') then
if isconst then
error("Cannot assign none to constant variable at line: "..name.line.." at column: "..name.col)
end
return {constant = false, value= "none", identifier= name, nodetype=7}
end
self:_advance()--advance equal sign
local val = self:_parse_assignexpr()
assert(self:_cur():of(6), "Expected a semicolon after variable declaration at line: "..name.line.." at column: "..name.col)
self:_advance()--skip semicolon
return {constant = isconst, value= val, identifier = name, nodetype= 7}
end
function parser:_parse_stmt(): {any}
if self:_cur():is(4, "private") or self:_cur():is(4, "public") or self:_cur():is(4, "static") or self:_cur():is(4, "sealed") then
elseif self:_cur():is(4, "let") or self:_cur():is(4, "const") then
print("in var dec!")
return self:_parse_var()
elseif self:_cur():is(4, "function") then
print("hoi")
elseif self:_cur():is(4, "class") then
print("cls")
elseif self:_cur():is(4, "event") then
print("event")
elseif self:_cur():is(4, "enum") then
print("enum")
else
print("assignexpr")
return self:_parse_assignexpr()
end
end
function parser:parse()
table.clear(self.result)
coroutine.wrap(function()
print("hello?")
print(#self.tokens)
while #self.tokens > 0 do
print("looped once?")
local stmt = self:_parse_stmt()
for i,v in pairs(stmt) do
print("[[")
print(i,v)
print("]]")
end
table.insert(self.result, stmt)
print(#self.tokens)
end
print("hi?")
if not self.onCompleted then return end
self.onCompleted()
end)()
end
return parser
it never prints hi 
this is the unit test:
local tokenizer_t = require(script.Parent.Modules.Compiler.Tokenizer)
local parser_t = require(script.Parent.Modules.Compiler.Parser)
local tokenizer = tokenizer_t.create([[
let x = 55;
let y = 77;
log(x + y);
]])
tokenizer.onCompleted = function()--add a callback to the completion of the tokenizer
local parser = parser_t.create(tokenizer.tokens)
print(#parser.tokens)
parser.onCompleted = function()
print("parser completed!")
for _, node in ipairs(parser.result) do
print("[[")
for i,v in pairs(node) do
print(i,v)
end
print("]]")
end
end
parser:parse()
print("after parser call!")
end
tokenizer:tokenize()--tokenize the source first
and this is the token class, if you are wondering what is of and holds do:
local token = {}
token.__tostring = function(self: {any})
return string.format("[line: %d, column: %d, type: %d, value: %s]", self.line, self.col, self.tokentype, tostring(self.value))
end
function token.new(tokentype: number, value: string, line: number, col: number): {any}
local new = setmetatable({__index = token}, token)
new.tokentype = tokentype
new.value = value
new.line = line
new.col = col
return new
end
function token:is(tokentype: number, value: any): boolean
return tokentype == self.tokentype and value == self.value
end
function token:of(tokentype: number): boolean
return tokentype == self.tokentype
end
function token:holds(value: any): boolean
return value == self.value
end
return token
edit: here is a picutre of the output:
end edit
why does the loop end?
thanks in advance!