Hello guys. Today I have started making my game’s save system. As you know, DataStores have limit of 4MB (4194304 bytes).
The problem I’m facing is that people may build like VERY big builds, with hundred thousands of parts. And I need to store/load them.
After reading about new buffer datatype support, I may say that it’s not good variant for me, because it uses memory in ratio 3:4 (usefull:actual). So amount of data which I can store with them is only 3MB.
On the other hand, there’s string support. Strings are encoded like sequence of bits. But, I have 1 problem with them too:
01111111 - valid symbol " "
1000000 - invalid symbol "�"
As I understood, that’s because roblox uses utf-8 and not ASCII. But this leads me to question: How I can make 8th bit usable?
1 Like
try using base64 to store data
found this script inside one of my veryyyyyy old folders, don’t really know who or what made it
(put this into a modulescript inside replicatedstorage, call it b64(or smth like that), and you’re good to go)
local base64 = {}
local extract = bit32 and bit32.extract -- lua 5.2/lua 5.3 in compatibility mode
if not extract then
if bit or _G.bit then -- luajit
local shl, shr, band = _G.bit.lshift, _G.bit.rshift, _G.bit.band
extract = function( v, from, width )
return band( shr( v, from ), shl( 1, width ) - 1 )
end
elseif _VERSION == "Luau" then -- roblox compatible?..
extract = function( v, from, width )
local w = 0
local flag = 2^from
for i = 0, width-1 do
local flag2 = flag + flag
if v % flag2 >= flag then
w = w + 2^i
end
flag = flag2
end
return w
end
else -- Lua 5.3+
extract = loadstring[[return function( v, from, width )
return ( v >> from ) & ((1 << width) - 1)
end]]()
end
end
function base64.makeencoder( s62, s63, spad )
local encoder = {}
-- yandere dev blessing :pray:
for b64code, char in pairs{[0]='A','B','C','D','E','F','G','H','I','J',
'K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y',
'Z','a','b','c','d','e','f','g','h','i','j','k','l','m','n',
'o','p','q','r','s','t','u','v','w','x','y','z','0','1','2',
'3','4','5','6','7','8','9',s62 or '+',s63 or'/',spad or'='} do
encoder[b64code] = char:byte()
end
return encoder
end
function base64.makedecoder( s62, s63, spad )
local decoder = {}
for b64code, charcode in pairs( base64.makeencoder( s62, s63, spad )) do
decoder[charcode] = b64code
end
return decoder
end
local DEFAULT_ENCODER = base64.makeencoder()
local DEFAULT_DECODER = base64.makedecoder()
local char, concat = string.char, table.concat
function base64.encode(str, encoder, usecaching)
encoder = encoder or DEFAULT_ENCODER -- use the default encoder if not provided
local t, k, n = {}, 1, #str -- new bytes
local lastn = n % 3 -- calculate the number of characters in the last group
local cache = {} -- cache table for performance optimization
-- iterate over the input string in groups of 3 characters
for i = 1, n - lastn, 3 do
local a, b, c = str:byte(i, i + 2) -- get the ascii codes of the characters
local v = a * 0x10000 + b * 0x100 + c -- combine the ascii codes into a single value
-- on this line lies a memory leak
local s
if usecaching then
s = cache[v] -- check if the value is already cached
if not s then
-- convert value to base64 characters and cache the result
s = char(encoder[extract(v, 18, 6)], encoder[extract(v, 12, 6)], encoder[extract(v, 6, 6)], encoder[extract(v, 0, 6)])
cache[v] = s
end
else
-- convert value to base64 characters without caching
s = char(encoder[extract(v, 18, 6)], encoder[extract(v, 12, 6)], encoder[extract(v, 6, 6)], encoder[extract(v, 0, 6)])
end
t[k] = s -- store b64 chrs in {}
k = k + 1 -- increment the tbl idx
end
-- handle the last group of characters (if any)
if lastn == 2 then
local a, b = str:byte(n - 1, n)
local v = a * 0x10000 + b * 0x100
t[k] = char(encoder[extract(v, 18, 6)], encoder[extract(v, 12, 6)], encoder[extract(v, 6, 6)], encoder[64])
elseif lastn == 1 then
local v = str:byte(n) * 0x10000
t[k] = char(encoder[extract(v, 18, 6)], encoder[extract(v, 12, 6)], encoder[64], encoder[64])
end
return concat(t)
end
function base64.isValidBase64(str)
local pattern = "[A-Za-z0-9+/=]"
return string.find(str, pattern) == 1 and string.match(str, pattern .. "*$") == str
end
function base64.decode(b64, decoder, usecaching)
decoder = decoder or DEFAULT_DECODER
local pattern = '[^%w%+%/%=]'
if decoder then
local s62, s63
for charcode, b64code in pairs(decoder) do
if b64code == 62 then s62 = charcode
elseif b64code == 63 then s63 = charcode
end
end
pattern = ('[^%%w%%%s%%%s%%=]'):format(char(s62), char(s63))
end
b64 = b64:gsub(pattern, '')
if not base64.isValidBase64(b64) then
return nil, "Invalid base64 string"
end
local cache = usecaching and {}
local t, k = {}, 1
local n = #b64
local padding = b64:sub(-2) == '==' and 2 or b64:sub(-1) == '=' and 1 or 0
for i = 1, padding > 0 and n - 4 or n, 4 do
-- decode each group of four base64 characters
local a, b, c, d = b64:byte(i, i + 3)
local s
if usecaching then
local v0 = a * 0x1000000 + b * 0x10000 + c * 0x100 + d
s = cache[v0]
if not s then
local v = (decoder[a] or 0) * 0x40000 + (decoder[b] or 0) * 0x1000 + (decoder[c] or 0) * 0x40 + (decoder[d] or 0)
s = char(extract(v, 16, 8), extract(v, 8, 8), extract(v, 0, 8))
cache[v0] = s
end
else
local v = (decoder[a] or 0) * 0x40000 + (decoder[b] or 0) * 0x1000 + (decoder[c] or 0) * 0x40 + (decoder[d] or 0)
s = char(extract(v, 16, 8), extract(v, 8, 8), extract(v, 0, 8))
end
t[k] = s
k = k + 1
end
if padding == 1 then
local a, b, c = b64:byte(n - 3, n - 1)
local v = (decoder[a] or 0) * 0x40000 + (decoder[b] or 0) * 0x1000 + (decoder[c] or 0) * 0x40
t[k] = char(extract(v, 16, 8), extract(v, 8, 8))
elseif padding == 2 then
local a, b = b64:byte(n - 3, n - 2)
local v = (decoder[a] or 0) * 0x40000 + (decoder[b] or 0) * 0x1000
t[k] = char(extract(v, 16, 8))
end
return table.concat(t)
end
-- programmers don't need sleep
-- better die while coding :muscle:
return base64
edit:
encode usage:
local b64 = require(game:GetService("ReplicatedStorage").b64)
local enc = b64.encode("data here")
print(enc) --> ZGF0YSBoZXJl
decode usage:
local b64 = require(game:GetService("ReplicatedStorage").b64)
local dec = b64.decode("ZGF0YSBoZXJl")
print(dec) --> data here