fully rewrited today
local Lexer = {}
Lexer.__index = Lexer
local Token = {}
Token.__index = Token
function Token.new(type, value)
return setmetatable({type = type, value = value}, Token)
end
function Lexer.new()
local lexer = setmetatable({
tokens = {},
patterns = {}
}, Lexer)
lexer:addToken("number", "%d+", function(value) return Token.new("Number", tonumber(value)) end)
lexer:addToken("operator", "%*", function() return Token.new("operator", "Asterisk") end)
lexer:addToken("operator", "%/", function() return Token.new("operator", "Slash") end)
lexer:addToken("operator", "%+", function() return Token.new("operator", "Plus") end)
lexer:addToken("operator", "%-", function() return Token.new("operator", "Minus") end)
lexer:addToken("whitespace", "%s+", function() return nil end)
return lexer
end
function Lexer:addToken(type, pattern, handler)
table.insert(self.patterns, {type = type, pattern = pattern, handler = handler})
end
function Lexer:tokenize(input)
if input == "" then return "" end
local position = 1
local length = #input
local _output = {}
while position <= length do
local matched = false
for _, tokenPattern in ipairs(self.patterns) do
local pattern = tokenPattern.pattern
local match = input:sub(position):match("^(" .. pattern .. ")")
if match then
local token = tokenPattern.handler(match)
if token then
table.insert(_output, "[" .. token.type .. ": " .. token.value .. "]")
end
position = position + #match
matched = true
break
end
end
if not matched then
error("Неизвестный символ: " .. input:sub(position, position))
end
end
local output = table.concat(_output, ", ")
return output
end
return Lexer