The only diffirence here is +-14 value passed to generate character. And using not %64 but %96.
And it’s not 8%, it’s 33.3%
1 Like
A 33% improvement of Base96 over Base64 is a common misconception.
Base64 uses log2(64) = 6 bits per byte, while base96 theoretically uses log2(96) ~= 6.58 bits per byte. The theoretical efficiency of base96 over base64 is 6.58/6 ~= 1.10, or about 10% better, not 33%.
1 Like
I found a pretty decent utf8-encoding implementation, lets benchmark it.
--!strict
--!optimize 2
--!native
-- DISCLAMER: stolen from https://github.com/cipharius/msgpack-luau/ for testing purposes
local function utf8Encode(message: string): string
local messageLength = #message
local nBytes = math.ceil(messageLength * (8 / 7))
local result = buffer.create(nBytes)
local bitPointer = 0
for i=1,nBytes do
local j = 1 + math.floor(bitPointer / 8)
local bitRemainder = bitPointer % 8
local byte = string.byte(message, j)
if bitRemainder == 0 then
buffer.writeu8(result, i-1, bit32.extract(byte, 1, 7))
elseif bitRemainder == 1 then
buffer.writeu8(result, i-1, bit32.extract(byte, 0, 7))
else
local nextByte = string.byte(message, j+1) or 0
buffer.writeu8(result, i-1, bit32.bor(
bit32.lshift(bit32.extract(byte, 0, 8 - bitRemainder), bitRemainder - 1),
bit32.extract(nextByte, 9 - bitRemainder, bitRemainder - 1)
))
end
bitPointer += 7
end
return buffer.tostring(result)
end
local function utf8Decode(message: string): string
local nBytes = math.floor(#message * 7 / 8)
local result = buffer.create(nBytes)
local bitPointer = 0
for i=1,nBytes do
local bitRemainder = bitPointer % 7
local byte = string.byte(message, 1 + math.floor(bitPointer / 7))
local nextByte = string.byte(message, 2 + math.floor(bitPointer / 7))
buffer.writeu8(result, i-1, bit32.bor(
bit32.lshift(bit32.extract(byte, 0, 7 - bitRemainder), bitRemainder + 1),
bit32.extract(nextByte, 6 - bitRemainder, 1 + bitRemainder)
))
bitPointer += 8
end
return buffer.tostring(result)
end
return {
utf8Encode = utf8Encode,
utf8Decode = utf8Decode,
}
Results: 16% less space and 5 times slower.
-- +----------------------------+-----------+----------+----------+
-- | # Encode |
-- +----------------------------+-----------+----------+----------+
-- | 🍎 32.0 KB utf8 encode | μ 1 | 723 μs | 74.9 KB | gc: 12 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍏 32.0 KB base64 encode | μ 1 | 149 μs | 44.1 KB | gc: 13 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍎 256.0 KB utf8 encode | μ 1 | 5.9 ms | 599.2 KB | gc: 10 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍏 256.0 KB base64 encode | μ 1 | 1.3 ms | 961.2 KB | gc: 12 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍎 2.0 MB utf8 encode | μ 1 | 48.9 ms | 4.8 MB | gc: 158 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍏 2.0 MB base64 encode | μ 1 | 9.5 ms | 7.7 MB | gc: 116 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍎 16.0 MB utf8 encode | μ 1 | 388.8 ms | 38.3 MB | gc: 1.3 ms
-- +----------------------------+-----------+----------+----------+
-- | 🍏 16.0 MB base64 encode | μ 1 | 73.7 ms | 61.5 MB | gc: 315 μs
-- +----------------------------+-----------+----------+----------+
-- | # Decode |
-- +----------------------------+-----------+----------+----------+
-- | 🍎 36.6 KB utf8 decode | μ 1 | 822 μs | 32.8 KB | gc: 15 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍏 42.7 KB base64 decode | μ 1 | 169 μs | 0.0 B |
-- +----------------------------+-----------+----------+----------+
-- | 🍎 292.6 KB utf8 decode | μ 1 | 5.4 ms | 262.2 KB | gc: 14 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍏 341.3 KB base64 decode | μ 1 | 1.3 ms | 0.0 B |
-- +----------------------------+-----------+----------+----------+
-- | 🍎 2.3 MB utf8 decode | μ 1 | 45.4 ms | 2.1 MB | gc: 122 μs
-- +----------------------------+-----------+----------+----------+
-- | 🍏 2.7 MB base64 decode | μ 1 | 9.6 ms | 0.0 B |
-- +----------------------------+-----------+----------+----------+
-- | 🍎 18.3 MB utf8 decode | μ 1 | 374.5 ms | 16.8 MB | gc: 1.2 ms
-- +----------------------------+-----------+----------+----------+
-- | 🍏 21.3 MB base64 decode | μ 1 | 75.6 ms | 0.0 B |
-- +----------------------------+-----------+----------+----------+
```