Recently the functions string.pack, string.unpack, and string.packsize were backported. I thought about using them to cut down on the size of saved data. Even if you can’t save binary data in datastores it could help when sending the information to clients (presumably sending a table across the network uses JSON?).
The data I am storing could be represented with the type
type T = number|boolean|string|{[number]:T}
This is what I used to convert the table to a string:
-- types:
-- 0 is table start
-- 1 is table end
-- 2 is boolean
-- 3 is string
-- 4 is 32 bit float
-- 5 is 64 bit float
-- >= 6 are for integers
local EncodingFunctions
local function EncodeBool(Value)
return string.char(2,Value and 1 or 0)
end
local function EncodeString(Value)
return "\x03"..string.pack("z",Value)
end
local function EncodeNumber(Value)
if Value%1 == 0 then
if Value < 0 then
local NValue = -Value
if NValue <= 0xFF then
return "\x0E"..string.pack("I1",NValue)
elseif NValue <= 0xFFFF then
return "\x0F"..string.pack("I2",NValue)
elseif NValue <= 0xFFFFFF then
return "\x10"..string.pack("I3",NValue)
elseif NValue <= 0xFFFFFFFF then
return "\x11"..string.pack("I4",NValue)
elseif NValue <= 0xFFFFFFFFFF then
return "\x12"..string.pack("I5",NValue)
elseif NValue <= 0xFFFFFFFFFFFF then
return "\x13"..string.pack("I6",NValue)
elseif NValue <= 0xFFFFFFFFFFFFF8 then
return "\x14"..string.pack("I7",NValue)
elseif NValue <= 0xFFFFFFFFFFFFF800 then
return "\x15"..string.pack("I8",NValue)
end
else
if Value <= 0xFF then
return "\x06"..string.pack("I1",Value)
elseif Value <= 0xFFFF then
return "\x07"..string.pack("I2",Value)
elseif Value <= 0xFFFFFF then
return "\x08"..string.pack("I3",Value)
elseif Value <= 0xFFFFFFFF then
return "\x09"..string.pack("I4",Value)
elseif Value <= 0xFFFFFFFFFF then
return "\x0A"..string.pack("I5",Value)
elseif Value <= 0xFFFFFFFFFFFF then
return "\x0B"..string.pack("I6",Value)
elseif Value <= 0xFFFFFFFFFFFFF8 then
return "\x0C"..string.pack("I7",Value)
elseif Value <= 0xFFFFFFFFFFFFF800 then
return "\x0D"..string.pack("I8",Value)
end
end
end
local Result = string.pack("f",Value)
if string.unpack("f",Result) == Value or Value ~= Value then
return "\x04"..Result
end
return "\x05"..string.pack("d",Value)
end
local function EncodeTable(Value)
local List = {"\x00"}
for _,v in ipairs(Value) do
table.insert(List,EncodingFunctions[type(v)](v))
end
table.insert(List,"\x01")
return table.concat(List)
end
EncodingFunctions = {boolean=EncodeBool,number=EncodeNumber,string=EncodeString,table=EncodeTable}
local function Encode(Value)
return EncodingFunctions[type(Value)](Value)
end
local function _Decode(Value,Position)
local Type = string.byte(Value,Position)
if Type == 0 then
local Starting = Position
local Table = {}
Position += 1
Type = string.byte(Value,Position)
while Type ~= 1 do
local Result,DataSize = _Decode(Value,Position)
table.insert(Table,Result)
Position += DataSize+1
Type = string.byte(Value,Position)
end
return Table,Position-Starting
elseif Type >= 6 then
if Type >= 14 then
local Length = Type-13
return -string.unpack(string.format("I%i",Length),Value,Position+1),Length
else
local Length = Type-5
return string.unpack(string.format("I%i",Length),Value,Position+1),Length
end
elseif Type == 4 then
return string.unpack("f",Value,Position+1),4
elseif Type == 5 then
return string.unpack("d",Value,Position+1),8
elseif Type == 3 then
local String = string.unpack("z",Value,Position+1)
return String,#String+1
else
return string.byte(Value,Position+1) ~= 0,1
end
end
local function Decode(Value)
return (_Decode(Value,1))
end
Is there a better way to encode a value of the type described above?
Currently the size of every element when encoding:
| Type | Size |
|---|---|
| Boolean | 2 bytes |
| Number | 2 to 9 bytes |
| String | Length+2 bytes |
| Table | 2 bytes |
| Strings are null terminated which means no nulls can be saved in them (which is fine for what I’m doing) | |
| Numbers can vary in size, if the number is an integer it will try to save it as an integer instead of a float. If it can’t save it as an integer or the number is not an integer, it will first try to save as a float, and if that loses some precision it will save as a double. |
I could use a format specific to the data that I’m using, which would probably be fine assuming I’m never saving the data (would have to keep a decoder for every version around).
