Help needed fixing visual artifacts of real-time syntax highlighter

okay, so, i have been using this syntax highlighter by NiceBuild1
but, it has some significant flaws
to begin with, the way how it gets the colors is completely inaccurate so i made significant changes to that
next to that, it did not support string interpolation local a = 5; print(`number is {a}`) so i had to implement that myself
next, i had to tackle gpt-generated code since that constantly contained a character that shifted the highlighted text downwards by 1, so i fixed that as well
concatenation and commas were broken from the start, and i have no idea how to fix it

code used:

local broken = "hello".."world"
local normal = "hello" .. "world"

local broken = "hello", "world"
local normal = "hello" , "world"

then we have THIS kind of a mess (html entities)
image

local a = "&lt"
local b = "<"

image

having that automatically completes it to < or >, or any kind of html entity you can use

so if anyone can help me fix this, thank you.

Modified highlighter module
Colors module
local Colors = {
	Text = Color3.fromRGB(204, 204, 204),
	Background = Color3.fromRGB(37, 37, 37),
	Selection = Color3.fromRGB(255, 255, 255),
	SelectionBackground = Color3.fromRGB(11, 90, 175),
	Operator = Color3.fromRGB(204, 204, 204),
	Number = Color3.fromRGB(255, 198, 0),
	String = Color3.fromRGB(173, 241, 149),
	Comment = Color3.fromRGB(102, 102, 102),
	Keyword = Color3.fromRGB(248, 109, 124),
	Error = Color3.fromRGB(255, 0, 0),
	Warning = Color3.fromRGB(255, 115, 21),
	Information = Color3.fromRGB(79, 79, 79),
	Hint = Color3.fromRGB(0, 139, 219),
	FindSelectionBackground = Color3.fromRGB(141, 118, 0),
	MatchingWordBackground = Color3.fromRGB(85, 85, 85),
	BuiltInFunction = Color3.fromRGB(132, 214, 247),
	Whitespace = Color3.fromRGB(85, 85, 85),
	CurrentLineHighlight = Color3.fromRGB(45, 50, 65),
	DebuggerCurrentLine = Color3.fromRGB(42, 60, 76),
	DebuggerErrorLine = Color3.fromRGB(76, 42, 42),
	LocalMethod = Color3.fromRGB(253, 251, 175),
	LocalProperty = Color3.fromRGB(97, 161, 241),
	Nil = Color3.fromRGB(255, 198, 0),
	Bool = Color3.fromRGB(255, 198, 0),
	FunctionKeyword = Color3.fromRGB(248, 109, 124),
	LocalKeyword = Color3.fromRGB(248, 109, 124),
	SelfKeyword = Color3.fromRGB(248, 109, 124),
	LuauKeyword = Color3.fromRGB(248, 109, 124),
	FunctionName = Color3.fromRGB(253, 251, 172),
	TODO = Color3.fromRGB(102, 102, 102),
	ScriptRuler = Color3.fromRGB(102, 102, 102),
	ScriptBracket = Color3.fromRGB(204, 204, 204),
	Type = Color3.fromRGB(0, 255, 255),
	DocumentationCodeBlockBackground = Color3.fromRGB(66, 66, 66),
	MenuPrimaryText = Color3.fromRGB(255, 255, 255),
	MenuSecondaryText = Color3.fromRGB(102, 102, 102),
	MenuSelectedText = Color3.fromRGB(255, 255, 255),
	MenuItemBackground = Color3.fromRGB(46, 46, 46),
	SelectedMenuItemBackground = Color3.fromRGB(38, 90, 169),
	ScriptEditorScrollbarBackground = Color3.fromRGB(46, 46, 46),
	ScriptEditorScrollbarHandle = Color3.fromRGB(179, 179, 179),
	IndentationRuler = Color3.fromRGB(102, 102, 102)
}

return Colors
--!native
--!optimize 2

local Colors = require(script.Colors)

local highlighter = {}
local keywords = {
	lua = {"and","break","or","else","elseif","if","then","until","repeat","while","do","for","in","end","local","return","function","export", "not"},
	rbx = {"game","workspace","script","math","string","table","task","wait","select","next","Enum","error","warn","tick","assert","shared","loadstring","tonumber","tostring","type","typeof","unpack","print","Instance","CFrame","Vector3","Vector2","Color3","UDim","UDim2","Ray","BrickColor","OverlapParams","RaycastParams","Axes","Random","Region3","Rect","TweenInfo","collectgarbage","utf8","pcall","xpcall","_G","setmetatable","getmetatable","os","pairs","ipairs"},
	operators = {"#","+","-","*","%","/","^","=","~","=","<",">",",",".","(",")","{","}","[","]",";",":"}
}

local function createKeywordSet(keywords)
	local set = {}
	for _, k in ipairs(keywords) do set[k] = true end
	return set
end

local luaSet = createKeywordSet(keywords.lua)
local rbxSet = createKeywordSet(keywords.rbx)
local operatorsSet = createKeywordSet(keywords.operators)

local function getHighlight(tokens, index)
	local token = tokens[index]
	if token == "true" or token == "false" then return Colors.Bool
	elseif tonumber(token) then return Colors.Number
	elseif token == "nil" then return Colors.Nil
	elseif token:sub(1,2) == "--" then return Colors.Comment
	elseif operatorsSet[token] then return Colors.Operator
	elseif luaSet[token] then return Colors.LuauKeyword
	elseif rbxSet[token] then return Colors.BuiltInFunction
	elseif token:sub(1,1) == "\"" or token:sub(1,1) == "\'" or token:sub(1,1) == "`" then return Colors.String
	end
	if tokens[index + 1] == "(" then
		if tokens[index - 1] == ":" then return Colors.FunctionName end
		return Colors.FunctionName
	end
	if tokens[index - 1] == "." then
		if tokens[index - 2] == "Enum" then return Colors.Keyword end
		return Colors.LocalProperty
	end
end

local function tokenizeInterpolation(str)
	local tokens, current, i = {}, "", 1
	while i <= #str do
		local c = str:sub(i,i)
		if c:match("[%w_]") then
			current = current .. c
		elseif operatorsSet[c] or c == "{" or c == "}" then
			if #current > 0 then table.insert(tokens, current) current = "" end
			table.insert(tokens, c)
		else
			if #current > 0 then table.insert(tokens, current) current = "" end
			table.insert(tokens, c)
		end
		i += 1
	end
	if #current > 0 then table.insert(tokens, current) end
	return tokens
end

function highlighter.run(source)
	local tokens, currentToken = {}, ""
	local inString, inComment, commentPersist = false, false, false
	local i = 1

	while i <= #source do
		local c = source:sub(i,i)

		if inComment then
			currentToken = currentToken .. c
			if (not commentPersist and c == "\n") or (commentPersist and source:sub(i-1,i) == "]]") then
				table.insert(tokens, currentToken)
				currentToken = ""
				inComment = false
				commentPersist = false
			end
			i += 1

		elseif inString then
			currentToken = currentToken .. c
			if c == inString and source:sub(i-1,i-1) ~= "\\" then
				table.insert(tokens, currentToken)
				currentToken = ""
				inString = false
				i += 1
			elseif c == "{" and inString == "`" then
				local preString = currentToken:sub(1,-2)
				if #preString > 0 then table.insert(tokens, preString) end
				table.insert(tokens, "{")
				currentToken = ""
				local braceLevel = 1
				local innerContent = ""
				i += 1
				while i <= #source and braceLevel > 0 do
					local innerChar = source:sub(i,i)
					if innerChar == "{" then
						braceLevel += 1
					elseif innerChar == "}" then
						braceLevel -= 1
					end
					if braceLevel > 0 then innerContent = innerContent .. innerChar end
					i += 1
				end
				local innerTokens = tokenizeInterpolation(innerContent)
				for _, t in ipairs(innerTokens) do table.insert(tokens, t) end
				table.insert(tokens, "}")
			else
				i += 1
			end

		else
			if source:sub(i,i+1) == "--" then
				if #currentToken > 0 then table.insert(tokens, currentToken) currentToken = "" end
				inComment = true
				commentPersist = source:sub(i+2,i+3) == "[["
				currentToken = "--"
				i += 2
			elseif c == "\"" or c == "\'" or c == "`" then
				if #currentToken > 0 then table.insert(tokens, currentToken) currentToken = "" end
				inString = c
				currentToken = c
				i += 1
			elseif operatorsSet[c] then
				if #currentToken > 0 then table.insert(tokens, currentToken) currentToken = "" end
				table.insert(tokens, c)
				i += 1
			elseif c:match("[%w_]") then
				currentToken = currentToken .. c
				i += 1
			else
				if #currentToken > 0 then table.insert(tokens, currentToken) currentToken = "" end
				table.insert(tokens, c)
				i += 1
			end
		end
	end

	if #currentToken > 0 then table.insert(tokens, currentToken) end

	local highlighted = {}
	local inBacktick = false

	for i, token in ipairs(tokens) do
		local color

		if token:sub(1,1) == "`" then
			inBacktick = not inBacktick
			color = Colors.String
		elseif (token == "{" or token == "}") and inBacktick then
			color = Colors.String
		else
			color = getHighlight(tokens,i)
		end

		if not color then
			color = Colors.Text
		end

		table.insert(highlighted,
			string.format("<font color=\"#%s\">%s</font>",
				color:ToHex(),
				token:gsub("<","&lt;"):gsub(">","&gt;"):gsub("\r", "")))

	end

	return table.concat(highlighted)
end


return highlighter

this did not really fix anything
the same issue still exists

I’m sure you will find something. Good luck.