Help on rewind system

I have a rewind system in my game that i am working on. The purpose of this system is to allow the server to be able to look back in time for the purpose of projectile hit validation. While it mostly seemed to be working fine, I find that there is a desync that occurs between 1-2 minutes into operation.

In the screenshot above, I have a system setup where I have a dummy moving back and forth across the baseplate. Pressing the P key makes a visual representation of a snapshot of the dummy’s movement. Blue represents what the client sees the moment the P key is pressed, and red represents what the server found when it looks back in time to the timestamp sent by the client. As you can see, some of them line up while others don’t. The ones that line up were taken before approximately one minute of operation, and the ones that do not line up were taken afterward.

I’ve noticed that the moment this desync occurs, the rig visibly rubberbands, moving backward a few studs. The distance of the desync remains consistent afterwards. Is this a studio issue or is there something wrong with my script?

--[[
Setup:
- Place this script in ServerScriptService
- Ensure workspace.Entities contains models with hitbox parts
--]]

local RunService = game:GetService("RunService")
local EntitiesFolder = workspace:WaitForChild("Entities")

-- Configuration
local MAX_SNAPSHOTS = 3000
local SnapshotInterval = 1 / 30 -- 30 Hz sampling

-- Data storage
local entitySnapshots = {} -- [entity] = { [index] = {timestamp = t, parts = { [part] = CFrame }} }
local snapshotIndex = 0

-- Utility: Get all BaseParts in a model
local function getHitboxParts(model)
	local parts = {}
	for _, part in ipairs(model:GetDescendants()) do
		if part:IsA("BasePart") then
			table.insert(parts, part)
		end
	end
	return parts
end

-- Utility: Record snapshot for all entities
local function recordSnapshot()
	snapshotIndex = (snapshotIndex + 1) % MAX_SNAPSHOTS
	local timestamp = workspace:GetServerTimeNow()

	for _, entity in ipairs(EntitiesFolder:GetChildren()) do
		if not entity:IsA("Model") then continue end
		local parts = getHitboxParts(entity)

		entitySnapshots[entity] = entitySnapshots[entity] or {}
		local snapshot = { timestamp = timestamp, parts = {} }

		for _, part in ipairs(parts) do
			snapshot.parts[part] = part.CFrame
		end

		entitySnapshots[entity][snapshotIndex] = snapshot
	end
end

-- Utility: Find closest snapshots before and after timestamp
local function findSnapshots(entity, timestamp)
	local snapshots = entitySnapshots[entity]
	if not snapshots then return nil end

	local before, after
	local minBeforeDist, minAfterDist = math.huge, math.huge

	for _, snap in pairs(snapshots) do
		local t = snap.timestamp
		if t <= timestamp and (timestamp - t) < minBeforeDist then
			before = snap
			minBeforeDist = timestamp - t
		elseif t > timestamp and (t - timestamp) < minAfterDist then
			after = snap
			minAfterDist = t - timestamp
		end
	end

	return before or after, after or before
end

-- Main Function: GetHitboxAtTimestamp
function GetHitboxAtTimestamp(entity, timestamp)
	local before, after = findSnapshots(entity, timestamp)
	if not before or not after then return nil end

	local alpha = (timestamp - before.timestamp) / (after.timestamp - before.timestamp)
	alpha = math.clamp(alpha, 0, 1)

	local hitboxModel = Instance.new("Model")
	hitboxModel.Name = "ReconstructedHitbox"
	hitboxModel.Parent = workspace

	for part, cfBefore in pairs(before.parts) do
		local cfAfter = after.parts[part]
		if not cfAfter then continue end

		local interpCF = cfBefore:Lerp(cfAfter, alpha)

		local clone = Instance.new("Part")
		clone.Size = part.Size
		clone.CFrame = interpCF
		clone.Anchored = true
		clone.CanCollide = false
		clone.Transparency = 0.5
		clone.Material = Enum.Material.Neon
		clone.BrickColor = BrickColor.new("Persimmon")
		clone.Parent = hitboxModel
	end

	return hitboxModel
end

-- Start recording snapshots
local timeAccumulator = 0
RunService.Heartbeat:Connect(function(dt)
	timeAccumulator += dt
	if timeAccumulator >= SnapshotInterval then
		recordSnapshot()
		timeAccumulator = 0
	end
	--accurate server time definition
	game.ReplicatedStorage.ServerTime.Value = game.Workspace:GetServerTimeNow()
end)

--test function to spawn clones
game.ReplicatedStorage.ServerRemotes.CLONE_communicate.OnServerEvent:Connect(function(player, humanoid, timestamp, brickcolor)
	timestamp -= .15
	local copy = GetHitboxAtTimestamp(humanoid.Parent, timestamp)
	local copy2 = GetHitboxAtTimestamp(humanoid.Parent, timestamp - .01)
	copy2 = GetHitboxAtTimestamp(humanoid.Parent, timestamp + .01)
	
	local time_Min = math.huge
	local time_Max = 0
	local totalEntry = 0
	for _, i in pairs(entitySnapshots[humanoid.Parent]) do
		if i.timestamp < time_Min then time_Min = i.timestamp end
		if i.timestamp > time_Max then time_Max = i.timestamp end 
		totalEntry += 1
	end
	
	print("Timestamp range: " .. tostring(time_Max - time_Min))
	print("Timestamp Count: " .. tostring(totalEntry))
end)

can you show me the video of the problem to better identify it?