StateSync - Multiplayer State Framework

[themediawantsyoudead]

StateSync - An open source state networking module

got tired of dealing with desync bugs and missed updates in my games so i made this.

basically handles all the annoying state sync stuff automatically. server stays in control, clients get updates, everyone stays synchronized without the usual headaches.

-- server
StateSync.CreateState("Game", { Status = "Waiting", Players = {} })
StateSync.UpdateState("Game", "Status", "InProgress")

-- client  
local GameState = StateSync.RequestState("Game")
StateSync.OnStateUpdate("Game", function(Name, Version, Snapshot, Ops)
    -- ...
end)

features:

• server authoritative (clients cant cheat)
• only sends diffs not full state (typically 80-90% smaller than full updates)
• missed updates get resent automatically every 2 seconds
• new players get current state instantly
• rate limiting built in (max 1 fetch per namespace per player every 2 seconds)

framework usage

server: CreateState(), UpdateState(), ListenStateChanges()
client: RequestState(), OnStateUpdate()

paths work like "PlayerCount" or {"Players", UserId, "Health"} for nested stuff.

why not remote events

they work until you need:

• new players to get existing state
• reliable delivery when network hiccups
• efficient updates (not sending everything constantly)
• more than like 5 different events

tried managing all that manually and its a pain. this just works.

resources

Main Module

local RunService = game:GetService("RunService")
local Players = game:GetService("Players")
local ReplicatedStorage = game:GetService("ReplicatedStorage")

local NewInstance = Instance.new

local Insert = table.insert
local Remove = table.remove
local Clone = table.clone
local Clear = table.clear

local Wait = task.wait
local Spawn = task.spawn

local GMatch = string.gmatch

local Floor = math.floor

local Server = RunService:IsServer()
local StateSync = {}

local RootFolder
local FetchRemote
local StreamRemote
local AckRemote

local SnapshotWindow = 64 -- history limit for diff generation
local ResendInterval = 2 -- seconds between resend attempts

do
	function CreateInstance(Type, Properties)
		local Properties = Properties or {}
		local Type = Type or "Folder"
		local NewObj = NewInstance(Type)

		for Property, Value in next, Properties do
			NewObj[Property] = Value
		end

		return NewObj
	end
end

do
	function DeepCopy(Value)
		local Result = {}

		if type(Value) ~= "table" then
			return Value
		end

		for K, V in next, Value do
			Result[DeepCopy(K)] = DeepCopy(V)
		end

		return Result
	end

	function TablesEqual(A, B)
		if A == B then
			return true
		end

		if type(A) ~= "table" or type(B) ~= "table" then
			return A == B
		end

		local Count = 0

		for K, V in next, A do
			Count += 1

			if not TablesEqual(V, B[K]) then
				return false
			end
		end

		for _ in next, B do
			Count -= 1

			if Count < 0 then
				return false
			end
		end

		return Count == 0
	end

	function PathClone(Path)
		local R = {}

		for I = 1, #Path do
			R[I] = Path[I]
		end

		return R
	end

	function NormalizePath(Path)
		if type(Path) == "string" then
			local Result = {}

			for Part in GMatch(Path, "[^%.]+") do
				local Num = tonumber(Part)
				Result[#Result + 1] = Num or Part
			end

			return Result
		elseif type(Path) == "table" then
			return PathClone(Path)
		else
			return {}
		end
	end

	function GetParentByPath(Root, Path)
		local Ref = Root

		if #Path == 0 then
			return nil, nil
		end

		for I = 1, #Path - 1 do
			if type(Ref) ~= "table" then
				return nil, nil
			end

			Ref = Ref[Path[I]]

			if type(Ref) ~= "table" then
				return nil, nil
			end
		end

		return Ref, Path[#Path]
	end

	function EnsurePath(Root, Path)
		local Ref = Root


		for I = 1, #Path - 1 do
			local K = Path[I]
			local NextRef = Ref[K]

			if type(NextRef) ~= "table" then
				NextRef = {}
				Ref[K] = NextRef
			end

			Ref = NextRef
		end

		return Ref, Path[#Path]
	end

	function SetValueByPath(Root, Path, Value)
		local Parent, Key = EnsurePath(Root, Path)
		Parent[Key] = Value
	end

	function ClearTable(T)
		Clear(T)
	end

	function AssignTable(Target, Source)
		ClearTable(Target)

		for K, V in next, Source do
			Target[K] = DeepCopy(V)
		end
	end

	function DiffTables(Old, New, BasePath, Ops)
		local OldIsTable = type(Old) == "table"
		local NewIsTable = type(New) == "table"
		local Seen = {}

		Ops = Ops or {}
		BasePath = BasePath or {}

		if Old == New then
			return Ops
		end

		if not OldIsTable or not NewIsTable then
			Insert(Ops, { Op = "set", Path = PathClone(BasePath), Value = DeepCopy(New) })
			return Ops
		end

		for K, OldV in next, Old do
			local NewV = New[K]
			local Path = PathClone(BasePath)

			Seen[K] = true
			Path[#Path + 1] = K

			if NewV == nil then
				Insert(Ops, { Op = "delete", Path = Path })
			else
				if type(OldV) == "table" and type(NewV) == "table" then
					DiffTables(OldV, NewV, Path, Ops)
				else
					if not TablesEqual(OldV, NewV) then
						Insert(Ops, { Op = "set", Path = Path, Value = DeepCopy(NewV) })
					end
				end
			end
		end

		for K, NewV in next, New do
			if not Seen[K] then
				local Path = PathClone(BasePath)
				Path[#Path + 1] = K

				Insert(Ops, { Op = "set", Path = Path, Value = DeepCopy(NewV) })
			end
		end

		return Ops
	end

	function ApplyOps(Target, Ops)
		if not Ops then
			return
		end

		if Ops.Reset then
			AssignTable(Target, Ops.Reset)
			return
		end

		for I = 1, #Ops do
			local Op = Ops[I]
			local Path = Op.Path

			if Op.Op == "set" then
				SetValueByPath(Target, Path, DeepCopy(Op.Value))
			elseif Op.Op == "delete" then
				local Parent, Key = GetParentByPath(Target, Path)

				if Parent then
					Parent[Key] = nil
				end
			end
		end
	end

	function ComposeFullReset(State)
		return { Reset = DeepCopy(State) }
	end

	function ValidateNamespace(Name)
		return type(Name) == "string" and #Name > 0 and #Name <= 64
	end

	function ValidateOps(Ops)
		if type(Ops) ~= "table" then
			return false
		end

		if Ops.Reset ~= nil then
			return type(Ops.Reset) == "table"
		end

		for I = 1, #Ops do
			local Op = Ops[I]

			if type(Op) ~= "table" then
				return false
			end

			if Op.Op ~= "set" and Op.Op ~= "delete" then
				return false
			end

			if type(Op.Path) ~= "table" then
				return false
			end
		end

		return true
	end
end

do
	local Initialized = false

	local StatesByNamespace = {}
	local PlayerIndex = {}

	function EnsureRemotes()
		if Initialized then
			return
		end

		RootFolder = ReplicatedStorage:FindFirstChild("StateSync")

		if not RootFolder then
			if Server then
				RootFolder = CreateInstance("Folder", {Name = "StateSync", Parent = ReplicatedStorage})
			else
				RootFolder = ReplicatedStorage:WaitForChild("StateSync")
			end
		end

		if Server then
			FetchRemote = CreateInstance("RemoteFunction", {Name = "Fetch", Parent = RootFolder})
			StreamRemote = CreateInstance("RemoteEvent", {Name = "Stream", Parent = RootFolder})
			AckRemote = CreateInstance("RemoteEvent", {Name = "Ack", Parent = RootFolder})
		else
			FetchRemote = RootFolder:WaitForChild("Fetch")
			StreamRemote = RootFolder:WaitForChild("Stream")
			AckRemote = RootFolder:WaitForChild("Ack")
		end

		Initialized = true
	end

	function GetEntry(Name)
		local Entry = StatesByNamespace[Name]

		if not Entry then
			error("StateSync: missing namespace " .. Name)
		end

		return Entry
	end

	function MakeEntry(Name, Initial)
		local Entry = {
			Name = Name,
			Data = DeepCopy(Initial or {}),
			Version = 0,
			Snapshots = {},
			Listeners = {},
			LastDiff = nil,
		}

		Entry.Snapshots[1] = { V = 0, S = DeepCopy(Entry.Data) }

		StatesByNamespace[Name] = Entry

		return Entry
	end

	function SnapshotPush(Entry)
		local Snapshots = Entry.Snapshots

		Insert(Snapshots, { V = Entry.Version, S = DeepCopy(Entry.Data) })

		if #Snapshots > SnapshotWindow then
			Remove(Snapshots, 1)
		end
	end

	function SnapshotFind(Entry, Version)
		local Arr = Entry.Snapshots

		for I = #Arr, 1, -1 do
			local Item = Arr[I]

			if Item.V == Version then
				return Item.S
			end
		end

		return nil
	end

	function EmitLocal(Name, Version, Ops, StateRef)
		local Entry = StatesByNamespace[Name]
		local Listeners = Entry and Entry.Listeners

		if not Listeners then
			return
		end

		for I = 1, #Listeners do
			local Fn = Listeners[I]

			if type(Fn) == "function" then
				pcall(Fn, Name, Version, Ops, StateRef)
			end
		end
	end

	function SendToPlayer(Player, Name, Entry, FromVersion)
		local Current = Entry.Version
		local PayloadOps
		local Base

		if FromVersion == Current then
			return
		end

		if FromVersion >= 0 then
			Base = SnapshotFind(Entry, FromVersion)

			if Base then
				PayloadOps = DiffTables(Base, Entry.Data)
			end
		end

		-- fallback to full reset if no diff available
		if not PayloadOps or #PayloadOps == 0 then
			PayloadOps = ComposeFullReset(Entry.Data)
		end

		StreamRemote:FireClient(Player, {
			Namespace = Name,
			Version = Current,
			Ops = PayloadOps,
		})
	end

	function Broadcast(Name, Entry)
		for _, Player in Players:GetPlayers() do
			local Index = PlayerIndex[Player]
			local Known = -1

			if Index and Index[Name] and Index[Name].Ack then
				Known = Index[Name].Ack
			end

			SendToPlayer(Player, Name, Entry, Known)

			if Index then
				Index[Name] = Index[Name] or {}
				Index[Name].Sent = Entry.Version
			end
		end
	end

	function ResendLoop()
		while Wait(ResendInterval) do
			-- resend unacknowledged updates to all players
			for _, Player in Players:GetPlayers() do
				local Index = PlayerIndex[Player]

				if Index then
					for Name, Entry in pairs(StatesByNamespace) do
						local Status = Index[Name]
						local Acked = Status and Status.Ack or -1

						if Acked < Entry.Version then
							SendToPlayer(Player, Name, Entry, Acked)

							if Status then
								Status.Sent = Entry.Version
							end
						end
					end
				end
			end
		end
	end

	function OnPlayerAdded(Player)
		PlayerIndex[Player] = {}
	end

	function OnPlayerRemoving(Player)
		PlayerIndex[Player] = nil
	end

	do
		local ServerApi = {}

		function ServerApi.CreateState(Namespace, InitialData)
			EnsureRemotes()

			if not Server then
				error("StateSync: server API on client")
			end

			if not ValidateNamespace(Namespace) then
				error("StateSync: invalid namespace")
			end

			if StatesByNamespace[Namespace] then
				error("StateSync: namespace exists " .. Namespace)
			end

			local Entry = MakeEntry(Namespace, InitialData or {})

			return DeepCopy(Entry.Data)
		end

		function ServerApi.UpdateState(Namespace, Path, Value)
			local Entry = GetEntry(Namespace)
			local Prev = DeepCopy(Entry.Data)
			local Norm = NormalizePath(Path)
			local Ops

			EnsureRemotes()

			if not Server then
				error("StateSync: server API on client")
			end

			SetValueByPath(Entry.Data, Norm, DeepCopy(Value))

			if TablesEqual(Prev, Entry.Data) then
				return Entry.Version
			end

			Ops = DiffTables(Prev, Entry.Data)

			Entry.Version += 1
			Entry.LastDiff = Ops

			SnapshotPush(Entry)
			EmitLocal(Namespace, Entry.Version, Ops, Entry.Data)
			Broadcast(Namespace, Entry)

			return Entry.Version
		end

		function ServerApi.ListenStateChanges(Namespace, Callback)
			EnsureRemotes()

			if not Server then
				error("StateSync: server API on client")
			end

			local Entry = GetEntry(Namespace)

			Insert(Entry.Listeners, Callback)

			return function()
				for I = #Entry.Listeners, 1, -1 do
					if Entry.Listeners[I] == Callback then
						Remove(Entry.Listeners, I)
						break
					end
				end
			end
		end

		if Server then
			EnsureRemotes()

			local FetchCooldowns = {}
			local AckCooldowns = {}

			FetchRemote.OnServerInvoke = function(Player, Namespace)
				if type(Player) ~= "userdata" or not ValidateNamespace(Namespace) then
					return nil
				end

				-- rate limit: max 1 fetch per namespace per player every 2 seconds
				local PlayerId = Player.UserId
				local Key = PlayerId .. ":" .. Namespace
				local Now = os.time()

				if FetchCooldowns[Key] and Now - FetchCooldowns[Key] < 2 then
					return nil
				end

				FetchCooldowns[Key] = Now

				local Entry = StatesByNamespace[Namespace]

				if not Entry then
					return nil
				end

				PlayerIndex[Player] = PlayerIndex[Player] or {}
				PlayerIndex[Player][Namespace] = PlayerIndex[Player][Namespace] or { Ack = -1, Sent = -1 }

				return {
					Version = Entry.Version,
					State = DeepCopy(Entry.Data),
				}
			end

			AckRemote.OnServerEvent:Connect(function(Player, Payload)
				local PlayerId = Player.UserId
				local Now = os.time()

				if AckCooldowns[PlayerId] and Now - AckCooldowns[PlayerId] < 0.1 then
					return
				end
				AckCooldowns[PlayerId] = Now

				local Valid = type(Payload) == "table"
					and ValidateNamespace(Payload.Namespace)
					and type(Payload.Version) == "number"

				if not Valid then
					return
				end

				local Entry = StatesByNamespace[Payload.Namespace]

				if not Entry then
					return
				end

				local V = Floor(Payload.Version)

				if V < 0 or V > Entry.Version then
					return
				end

				PlayerIndex[Player] = PlayerIndex[Player] or {}
				PlayerIndex[Player][Payload.Namespace] = PlayerIndex[Player][Payload.Namespace] or { Ack = -1, Sent = -1 }
				PlayerIndex[Player][Payload.Namespace].Ack = V
			end)

			Players.PlayerAdded:Connect(OnPlayerAdded)
			Players.PlayerRemoving:Connect(OnPlayerRemoving)

			for _, P in Players:GetPlayers() do
				OnPlayerAdded(P)
			end

			Spawn(ResendLoop)

			StateSync.CreateState = ServerApi.CreateState
			StateSync.UpdateState = ServerApi.UpdateState
			StateSync.ListenStateChanges = ServerApi.ListenStateChanges
		end
	end
end

do
	local Initialized = false

	local CacheByNamespace = {}
	local VersionByNamespace = {}
	local CallbacksByNamespace = {}

	function EnsureClientRemotes()
		if Initialized then
			return
		end

		local Folder = ReplicatedStorage:WaitForChild("StateSync")

		RootFolder = Folder
		FetchRemote = Folder:WaitForChild("Fetch")
		StreamRemote = Folder:WaitForChild("Stream")
		AckRemote = Folder:WaitForChild("Ack")

		Initialized = true
	end

	function EnsureCache(Name)
		if not CacheByNamespace[Name] then
			CacheByNamespace[Name] = {}
			VersionByNamespace[Name] = -1
		end

		return CacheByNamespace[Name]
	end

	function SetVersion(Name, Version)
		VersionByNamespace[Name] = Version
	end

	function Emit(Name, Version, Ops)
		local Listeners = CallbacksByNamespace[Name]

		if not Listeners then
			return
		end

		for I = 1, #Listeners do
			local Fn = Listeners[I]

			if type(Fn) == "function" then
				pcall(Fn, Name, Version, DeepCopy(CacheByNamespace[Name]), Ops)
			end
		end
	end

	if not Server then
		EnsureClientRemotes()

		StreamRemote.OnClientEvent:Connect(function(Payload)
			if type(Payload) ~= "table" then
				return
			end

			local Name = Payload.Namespace
			local Version = Payload.Version
			local Ops = Payload.Ops

			if not ValidateNamespace(Name) or type(Version) ~= "number" or not ValidateOps(Ops) then
				return -- drop malformed packets
			end

			local Cache = EnsureCache(Name)

			ApplyOps(Cache, Ops)
			SetVersion(Name, Version)
			AckRemote:FireServer({ Namespace = Name, Version = Version })
			Emit(Name, Version, Ops)
		end)

		function StateSync.RequestState(Namespace)
			local Response = FetchRemote:InvokeServer(Namespace)
			local Cache = EnsureCache(Namespace)

			EnsureClientRemotes()

			-- client side validation is just for UX, server will validate again
			if not ValidateNamespace(Namespace) then
				error("StateSync: invalid namespace")
			end

			if type(Response) ~= "table" or type(Response.State) ~= "table" or type(Response.Version) ~= "number" then
				error("StateSync: fetch failed for " .. Namespace)
			end

			AssignTable(Cache, Response.State)
			SetVersion(Namespace, Response.Version)
			AckRemote:FireServer({ Namespace = Namespace, Version = Response.Version })

			return DeepCopy(Cache)
		end

		function StateSync.OnStateUpdate(Namespace, Callback)
			EnsureClientRemotes()

			if not ValidateNamespace(Namespace) then
				error("StateSync: invalid namespace")
			end

			CallbacksByNamespace[Namespace] = CallbacksByNamespace[Namespace] or {}
			Insert(CallbacksByNamespace[Namespace], Callback)

			return function()
				local Arr = CallbacksByNamespace[Namespace]

				if not Arr then
					return
				end

				for I = #Arr, 1, -1 do
					if Arr[I] == Callback then
						Remove(Arr, I)
						break
					end
				end
			end
		end
	end
end

return StateSync

Server Demo

-- move into ServerScriptService

local ReplicatedStorage = game:GetService("ReplicatedStorage")
local Players = game:GetService("Players")

local StateSync = require(ReplicatedStorage:WaitForChild("StateSync"))

-- setup game states
do
	StateSync.CreateState("Game", { Status = "Waiting", Round = 1, Timer = 60 })
	StateSync.CreateState("Lobby", { Players = {} })
	StateSync.CreateState("PlayerStats", {})
end

-- handle players
do
	Players.PlayerAdded:Connect(function(Player)
		local UserId = tostring(Player.UserId)

		StateSync.UpdateState("Lobby", { "Players", UserId }, {
			Name = Player.Name,
			Level = math.random(1, 50),
			Ready = false
		})

		StateSync.UpdateState("PlayerStats", { UserId }, {
			Kills = 0,
			Deaths = 0, 
			Score = 0
		})
	end)

	Players.PlayerRemoving:Connect(function(Player)
		local UserId = tostring(Player.UserId)

		StateSync.UpdateState("Lobby", { "Players", UserId }, nil)
		StateSync.UpdateState("PlayerStats", { UserId }, nil)
	end)
end

-- game loop
do
	task.spawn(function()
		local States = { "Waiting", "Starting", "InProgress", "Ended" }
		local Current = 1

		while task.wait(5) do
			StateSync.UpdateState("Game", "Status", States[Current])
			StateSync.UpdateState("Game", "Timer", math.random(30, 300))

			if math.random() > 0.6 then -- 40% (chance for round to change)
				StateSync.UpdateState("Game", "Round", math.random(1, 10))
			end

			Current = Current % #States + 1
		end
	end)
end

Client Demo

-- move into StarterPlayerScripts

local ReplicatedStorage = game:GetService("ReplicatedStorage")
local Players = game:GetService("Players")

local StateSync = require(ReplicatedStorage:WaitForChild("StateSync"))
local Player = Players.LocalPlayer

-- get initial states
do
	local GameState = StateSync.RequestState("Game")
	local LobbyState = StateSync.RequestState("Lobby") 
	local StatsState = StateSync.RequestState("PlayerStats")

	print("Connected Successfully")
	print("Game status:", GameState.Status)
end

-- listen for updates
do
	StateSync.OnStateUpdate("Game", function(Name, Version, Snapshot)
		print("Game:", Snapshot.Status, "| Round", Snapshot.Round)
	end)

	StateSync.OnStateUpdate("Lobby", function(Name, Version, Snapshot)
		local Count = 0

		for _ in pairs(Snapshot.Players or {}) do 
			Count += 1 
		end

		print("Lobby:", Count, "players online")
	end)

	StateSync.OnStateUpdate("PlayerStats", function(Name, Version, Snapshot)
		local MyStats = Snapshot[tostring(Player.UserId)]

		if MyStats then
			print("My stats: Score", MyStats.Score, "K/D", MyStats.Kills .. "/" .. MyStats.Deaths)
		end
	end)
end

info

stress tested with 50 bots updating constantly:

Update #740 | Players: 50 | Rate: 27.1/sec | Status: InProgress

consistent 27-30 updates/sec with no performance issues.

[gwenniekins]

for missed updates, does it make sure it doesn’t update to a previous update? since thats where desync tends to occur when an update gets delayed and then overwrites a current one with an old update, sure since it’s server verified it would get rectified fairly soon, but is there any sort of protection to prevent it from occurring in the first place?

[themediawantsyoudead]

yeah it handles that. each update gets a version number that increments, so if an old update shows up late the client just ignores it since it already has a higher version. the resend system also tracks versions, it only resends what the client hasn’t acked yet, so you cant get that annoying bug where a delayed packet overwrites newer state.

[kneebandit49 (NoTaxEvading)]

@soldinfections

link to the module

I get an error page when I click the link

[themediawantsyoudead]

fixed, thanks for the info, check the post.