Binary Table Encoder V1

Model: http://www.roblox.com/Binary-Table-Encode-item?id=246360496

What it does is it creates a binary string from a table, and back.

Code:

[code]–Module by FromLegoUniverse
–[[----Functions:------
Module:EncodeTable(Table to encode, Use64Encode)

Encodes a table into a binary string that can be printed or stored in the DataStore service.
Use64Encode set to true will use 64 bit encoding for floats. Not true (false, nil, etc) will use 34 bit encoding
for floats. 64 bit is for accuracy (good for above 1), and 32 bit is smaller data size.
Returns a binary string that can be decoded

Module:DecodeTable(String to decode)
Decodes an encoded binary string back to a table.
Note: Use64Encode is stored in the string, and is not an arguement in this case.
Returns the decoded table
–]]

local BitBufferLocation = script
local BitBufferModule = BitBufferLocation:FindFirstChild(“BitBuffer”)
if not BitBufferModule then
for i = 1, 5*60 do
BitBufferModule = BitBufferLocation:FindFirstChild(“BitBuffer”)
if BitBufferModule then break end
wait()
end
if not BitBufferModule then error(“BitBuffer Module not found”) end
end

local BitBuffer = require(BitBufferModule)
local TypeIntegerLength = 3
local IntegerLength = 10

local function TypeToId(Type)
if Type == “Integer” then
return 1
elseif Type == “NegInteger” then
return 2
elseif Type == “Number” then
return 3
elseif Type == “String” then
return 4
elseif Type == “Boolean” then
return 5
elseif Type == “Table” then
return 6
end
return 0
end

local function IdToType(Type)
if Type == 1 then
return “Integer”
elseif Type == 2 then
return “NegInteger”
elseif Type == 3 then
return “Number”
elseif Type == 4 then
return “String”
elseif Type == 5 then
return “Boolean”
elseif Type == 6 then
return “Table”
end
end

local function IsInt(Number)
local Decimal = string.find(tostring(Number),“%.”)
if Decimal then
return false
else
return true
end
end

local function log(Base,Number)
return math.log(Number)/math.log(Base)
end

local abs = math.abs
local function GetMaxBitsInt(Table)
local Max = 0
for Key,Value in pairs(Table) do
if type(Value) == “number” then
Value = abs(Value)
if IsInt(Value) and Value > 0 then
local Bits = math.ceil(log(2,Value + 1))
if Bits > Max then Max = Bits end
end
end

	if type(Key) == "number" then
		Key = abs(Key)
		if IsInt(Key) and Value > 0 then
			local Bits = math.ceil(log(2,Key + 1))
			if Bits > Max then Max = Bits end
		end
	end
end
return Max*2

end

local function GetTableLength(Table)
local Total = 0
for , in pairs(Table) do
Total = Total + 1
end
return Total
end

local function GetType(Key)
local Type = type(Key)
if Type == “number” then
if IsInt(Key) then
if Key < 0 then
return “NegInteger”
end
return “Integer”
else
return “Number”
end
else
return Type
end
end

local function GetAllType(Table)
local Type
for Key,_ in pairs(Table) do
if not Type then
Type = GetType(Key)
end
if type(Key) ~= Type then
local NewType = GetType(Key)
if NewType ~= Type then
return nil
end
end
end
if Type == “Number” then
return “Number”
elseif Type == “Integer” then
return “Integer”
elseif Type == “NegInteger” then
return “NegInteger”
else
return “String”
end
end

local Module = {}
function Module:EncodeTable(Table,UseBase64)
local AllType = GetAllType(Table)
local Buffer = BitBuffer.Create()
if UseBase64 == true then
Buffer:WriteBool(true)
else
Buffer:WriteBool(false)
end
Buffer:WriteUnsigned(IntegerLength,GetTableLength(Table))

local function WriteFloat(Number)
	if UseBase64 == true then
		Buffer:WriteFloat64(Number)
	else
		Buffer:WriteFloat32(Number)
	end
end
Buffer:WriteUnsigned(TypeIntegerLength,TypeToId(AllType))
local MaxBits = GetMaxBitsInt(Table)
Buffer:WriteUnsigned(IntegerLength,MaxBits)

local function WriteKey(Key,AllowAllSame)
	if not (AllowAllSame == true and AllType) then
		Buffer:WriteUnsigned(TypeIntegerLength,Key)
	elseif AllowAllSame == false then
		Buffer:WriteUnsigned(TypeIntegerLength,Key)
	end
end

for Key,Value in pairs(Table) do
	if type(Key) == "string" then
		WriteKey(TypeToId("String"),true)
		Buffer:WriteString(Key)
	elseif type(Key) == "number" and IsInt(Key) then
		if Key >= 0 then
			WriteKey(TypeToId("Integer"),true)
			Buffer:WriteUnsigned(MaxBits,Key)
		else
			WriteKey(TypeToId("NegInteger"),true)
			Buffer:WriteSigned(MaxBits*2,Key)
		end
	elseif type(Key) == "number" then
		WriteKey(TypeToId("Number"),true)
		WriteFloat(Key)
	end
	
	if type(Value) == "boolean" then
		WriteKey(TypeToId("Boolean"))
		Buffer:WriteBool(Value)
	elseif type(Value) == "number" then
		if IsInt(Value) then
			if Value < 0 then
				WriteKey(TypeToId("NegInteger"))
				Buffer:WriteSigned(MaxBits*2,Value)
			else
				WriteKey(TypeToId("Integer"))
				Buffer:WriteUnsigned(MaxBits,Value)
			end
		else
			WriteKey(TypeToId("Number"))
			WriteFloat(Value)
		end
	elseif type(Value) == "table" then
		WriteKey(TypeToId("Table"))
		Buffer:WriteString(Module:EncodeTable(Value,UseBase64))
	else
		WriteKey(TypeToId("String"))
		Buffer:WriteString(tostring(Value))
	end
end
return Buffer:ToBase64()

end

function Module:DecodeTable(BinaryString)
local Buffer = BitBuffer.Create()
Buffer:FromBase64(BinaryString)
local Table = {}
local UseBase64 = Buffer:ReadBool()
local function ReadFloat()
if UseBase64 == true then
return Buffer:ReadFloat64()
else
return Buffer:ReadFloat32()
end
end
local Length = Buffer:ReadUnsigned(IntegerLength)
local AllType = Buffer:ReadUnsigned(TypeIntegerLength)
local MaxBits = Buffer:ReadUnsigned(IntegerLength)
if AllType == 0 then AllType = nil end

for i = 1, Length do
	local KeyType,Key = AllType or Buffer:ReadUnsigned(TypeIntegerLength)
	
	local KeyRealType = IdToType(KeyType)
	if KeyRealType == "Integer" then
		Key = Buffer:ReadUnsigned(MaxBits)
	elseif KeyRealType == "NegInteger" then
		Key = Buffer:ReadSigned(MaxBits*2)
	elseif KeyRealType == "Number" then
		Key = ReadFloat()
	elseif KeyRealType == "String" then
		Key = Buffer:ReadString()
	end
	
	local ValueType,Value = Buffer:ReadUnsigned(TypeIntegerLength)
	local ValueRealType = IdToType(ValueType)
	if ValueRealType == "String" then
		Value = Buffer:ReadString()
	elseif ValueRealType == "Boolean" then
		Value = Buffer:ReadBool()
	elseif ValueRealType == "Number" then
		Value = ReadFloat()
	elseif ValueRealType == "Integer" then
		Value = Buffer:ReadUnsigned(MaxBits)
	elseif ValueRealType == "NegInteger" then
		Value = Buffer:ReadSigned((MaxBits * 2))
	elseif ValueRealType == "Table" then
		Value = Module:DecodeTable(Buffer:ReadString())
	end
	
	Table[Key] = Value
end
return Table

end

return Module[/code]

Requires the BitBuffer module by Stravant by the way.
[strike]Userdata is not encoded nor decoded. I will work on that. :P[/strike] Edit: Done. If needed, I will put that code.

I did attempt a good amount of optimizations to minimize the size in some cases. And yes, you can have multiple types of keys (ex: number keys and string keys).

Please note, this is my first thing ever with relevance to binary strings (and also my first code with logarithms, now I feel like I actually can use something from my algebra class)

1 Like

Have you done any comparisons of the size of tables you can store using this vs. using DataStoreService’s regular JSON encoding?

Ok, so it is dependent because it is optimized one way, and not the other. Basically, mixed table are longer, but non-mixed are shorting.
Mixed:

{"Foo":2,"Version":"V.1","Numbers":[1,7,1,5,2,8],"Bar":true}
IASAEGWOENn1bLus85QYNPfOow1BLmEZzoDPfuaUhSDSc4QIYts85pf7GCyqrYQw473QEB

It basically encodes as:
[Number of values][Max bits for integers][0]
And for each value:
[Id for key (number, string, etc)][Key Name][Id for value (number, bool, table,etc][Value (encoding/decoding varies)]

Non-mixed keys:

{"Bar":true,"C":false,"B":5,"Foo":2,"Version":"V.1","Numbers":4,"A":3}
OAaACCRG4sebxllPHiQwaZ5zT/2NEkVXxggQIShhQUEGWOENjvfDREA

It basically encodes as:
[Number of values][Max bits for integers][Id for keys]
And for each value:
[Key Name][Id for value (number, bool, table,etc][Value (encoding/decoding varies)]

This was my first thing with bit stuff, so anything I can learn for optimizing size would be good.

Look for patterns, and create a pattern character that is used to replace all the areas that pattern is seen, thus reducing size on heavily redundant information (that binary has) this is called compression, you would store all your pattern characters at the top of the ‘file’, with their content right next to it, so it can be uncompressed later

Really Simplified Example:
File:
qwertyqwertyqwerty

PatternedFile:
xqwerty|aaa

This sounds like an NP-hard problem, what do you imagine the algorithm for this would look like?

This sounds like an NP-hard problem, what do you imagine the algorithm for this would look like?[/quote]

It doesn’t. It sounds like LZ-77 compression. LZ77 and LZ78 - Wikipedia

EDIT: here’s a better video - [url]https://www.youtube.com/watch?v=goOa3DGezUA[/url]

This sounds like an NP-hard problem, what do you imagine the algorithm for this would look like?[/quote]

It doesn’t. It sounds like LZ-77 compression. LZ77 and LZ78 - Wikipedia

EDIT: here’s a better video - [url]https://www.youtube.com/watch?v=goOa3DGezUA[/url][/quote]

Okay, provided the size of the lookahead is limited.