[R$10,000 Bounty] Random Data Loss

Hi everyone, I’m Bladian and I currently have a data loss issue from my data managing class.

Stats seem to either backtrack, or reset entirely, we have no knowledge on how it works, and why it happens.

local Module = {}
-- Settings --

Module.UseDefaultStatsInStudio = false
Module.ResetStatsInStudio = false
Module.ResetStatsInGame = false

-- Variables ---

local dataStoreService = game:GetService("DataStoreService")
local ServerStorage = game:GetService('ServerStorage')
local RunService = game:GetService('RunService')

--local dataStore = dataStoreService:GetDataStore("Random2")
local dataStore = dataStoreService:GetDataStore("Random2")

-- Functions --

function Module.insertPlayer(player)
	-- print("Inserted " .. player.Name .. " to the Data Manager")
	
	local defaultStats = {
		
		Stats = {
			
			Money = 0,
		
			XP = 0, 
			Level = 1,
		
			BackpackLevel = 1, 
			SpeedLevel = 1, 
			RangeLevel = 1,
			SpinHarvestLevel = 1, 
			
			Mined = 0,
			
			RobuxSpent = 0,
		
			MaxEquippedPets = 3,
			MaxInventoryPets = 20, --20,
			
			Version = 1.0,
			
			FirstJoin = true,
			
			TimePlayed = 0,
						
			TotalMoneyEarned = 0,
			GoldMined = 0,
			
			EggsHatched = 0,
			
			GodMinionGiven = false,
		},
		
		Backpack = {
			Wheat = 0,
			Harvest2 = 0,
			Harvest3 = 0,
		},
		
		RedeemedCodes = {},
		Gamepasses = {},
		UnlockedAreas = {"Valley"},
		Badges = {},
		Pets = {},
		UnlockedPets = {},
		UnlockedPickaxes = {},
		Pickaxes = {},

		Loaded = false,
		Spawned = false,
		
		CurrentKillStreak = 0,
		
		Rewards = {
			Daily = {
				Day = 0,
				HighestStreak = 0,
				LastRewardTime = 0,
			},
			DailySecond = {
				Day = 0,
				HighestStreak = 0,
				LastRewardTime = 0,
			},
			UnderHourly = {
				Streak = 0,
				HighestStreak = 0,
				LastRewardTime = 0,
			},
			Grouply = {
				Streak = 0,
				HighestStreak = 0,
				LastRewardTime = 0,
			},
		},
		
		Settings = {
			Music = true,
			Sounds = true,
			ShowMinions = true,
			ShowOtherMinions = true,
			MinionsPerClick = 1,
			HUDEffects = true,
			--MusicVolume = 0.5,
			FPSCounter = true,
			JoinNotification = true,
			BossNotification = true,
			BrawlNotification = true,
			ChatNotifications = true,
			MaxMagicNotification = true,
		},
		
	}

	Module[player.UserId] = defaultStats
	
	if not RunService:IsStudio() or not Module.UseDefaultStatsInStudio and not Module.ResetStatsInStudio then
		if Module.ResetStatsInGame then
			Module[player.UserId].Loaded = true
			Module[player.UserId].Spawned = false	
		else
			Module.getDataFromDataStore(player)
		end
		--dataStore:SetAsync(player.UserId, defaultStats)
		
	elseif Module.UseDefaultStatsInStudio or Module.ResetStatsInStudio then
		Module[player.UserId].Loaded = true
		Module[player.UserId].Spawned = false	
	end
	
end

function deepCopy(player, original, playerTable)
    local copy = {}
	
    for k, v in pairs(original) do
		
		if k == "Loaded" or k == "Spawned" then
		
		else
			if playerTable ~= nil and playerTable[k] ~= nil then
				if type(v) == "table" then
		        	copy[k] = deepCopy(player, v, playerTable[k])
				else
					copy[k] = playerTable[k]
	        	end 
				
			else
				if type(v) == "table" then
					local editedValue = nil
					if playerTable ~= nil then
						editedValue = playerTable[k]
					end
		        	copy[k] = deepCopy(player, v, playerTable[k])
				else
					copy[k] = v
	        	end 
			end
		end
        -- as before, but if we find a table, make sure we copy that too
		
--		print(k, v, copy[k])
	end
 	
	return copy
end

function nextDeepCopy(player, editedVersion, playerTable) 
	
	local copy = {}
	
	for k, v in pairs(playerTable) do
		if k == "Loaded" or k == "Spawned" then
		
		else
			if editedVersion ~= nil and editedVersion[k] ~= nil then
				if type(v) == "table" then
		        	copy[k] = nextDeepCopy(player, editedVersion[k], v)
				else
					copy[k] = v
				end
			else
				if type(v) == "table" then
					local editedValue = nil
					if editedVersion ~= nil then
						editedValue = editedVersion[k]
					end
		        	copy[k] = nextDeepCopy(player, editedValue, v)
				else
					copy[k] = v
				end
			end
		end
        -- as before, but if we find a table, make sure we copy that too
		
	end
	
	return copy
end

function tableMerge(t1, t2)
    for k,v in pairs(t2) do
        if type(v) == "table" then
            if type(t1[k] or false) == "table" then
                tableMerge(t1[k] or {}, t2[k] or {})
            else
                t1[k] = v
            end
        else
            t1[k] = v
        end
    end
    return t1
end

function Module.getDataFromDataStore(player, Attempts)
	
	if not player:IsDescendantOf(game:GetService('Players')) then return end

	local successData, currentStatsData = pcall(function()
		return dataStore:GetAsync(player.UserId)
	end)
	if successData then
		if currentStatsData == nil then
			--Module.updateStatsToDataStore(player)
			
			Module[player.UserId]["Loaded"] = true
			Module[player.UserId]["Spawned"] = false	
		else
			--Module[player.UserId] = currentStatsData
--			local json = game:GetService("HttpService"):JSONEncode(currentStatsData)
--			print("Here")
--			print(json)

			local editedVersion = deepCopy(player, Module[player.UserId], currentStatsData)
			local editedVersion2 = nextDeepCopy(player, editedVersion, currentStatsData)
			
			Module[player.UserId] = tableMerge(editedVersion, editedVersion2)
			--Module[player.UserId] = currentStatsData
			
			Module[player.UserId].Stats.MaxInventoryPets = 25
			
			Module[player.UserId].Loaded = true
			Module[player.UserId].Spawned = false	
			
			local playerData = Module.playerData(player)
			
			-- print("Finished fetching data for " .. player.Name, "("..player.UserId..")")
		end
	else
		spawn(function()
			error(currentStatsData)
		end)
		player:Kick('Data failed to load')
		--[[
		wait(1)
		Attempts = Attempts or 1
		return Attempts < 10 and Module.getDataFromDataStore(player, Attempts + 1)]]
	end
end

function Module.updateStatsToDataStore(player, Attempts)
	
	local playerId = player.UserId
	
	if not Module[playerId] or Module.UseDefaultStatsInStudio and RunService:IsStudio() then return end
	
	local Copy = {}
	for i, v in next, Module[playerId] do
		Copy[i] = v
	end
	
	Copy.Stats.Money = math.floor(Copy.Stats.Money + 0.5)
	Copy.Stats.HighestKillStreak = Copy.CurrentKillStreak
	--[[if Copy.JoinTime then
		Copy.Stats.TimePlayed = Copy.Stats.TimePlayed + (math.floor(tick() - Copy.JoinTime))
	end]]
	Copy.Loaded = false
	Copy.Spawned = false
	
	local success, err = pcall(function()
		
		-- print("Setting all the data now", player.Name .. "(" .. playerId .. ")")
		dataStore:SetAsync(player.UserId, Copy)
		
	end)
	if success == false then
		spawn(function()
			error(err)
		end)
		wait(1)
		Attempts = Attempts or 1
		return Attempts < 10 and Module.updateStatsToDataStore(player, Attempts + 1)
	else
		-- print("All data was set", playerId)
	end
	
end

function Module.removeStats(player) 
	
	Module[player.UserId] = nil 
	
end

function Module.boostActive(player, boostName) 
	
	local playerData = Module.playerData(player)
	if not playerData then return end
	return playerData.Boosts[boostName] > os.time()
	
end

function Module.playerDataLoaded(player)
	return Module[player.UserId] and Module.playerData(player).Loaded
end


function Module.playerData(player)

	if not Module[player.UserId] then
		repeat wait() until Module[player.UserId] or not game:GetService('Players'):FindFirstChild(player.Name)
	end
	
	return Module[player.UserId]
	
end

function Module.backpackAmount(player)
	
	if not Module[player.UserId] then
		repeat wait() until Module[player.UserId] or not game:GetService('Players'):FindFirstChild(player.Name)
	end
	
	local Amount = 0
	
	for _, CropAmount in next, Module[player.UserId].Backpack do
		Amount = Amount + CropAmount
	end
	
	return Amount
end

return Module

We currently use this class in these ways.

game.Players.PlayerAdded:Connect(function(player)
	
	DataManager.insertPlayer(player)
	DataManager[player.UserId].JoinTime = math.floor(tick())
	
end)
spawn(function()
	while true do 
		wait(60)
		for _, player in next, game.Players:GetPlayers() do
			if DataManager.playerDataLoaded(player) then
				DataManager.updateStatsToDataStore(player)
				print("Doing 60 second save for " .. player.Name)
			end
		end
	end
end)
if game:GetService('RunService'):IsStudio() then return end

game:BindToClose(function()
	for _, player in next, game:GetService("Players"):GetPlayers() do
		if DataManager.playerDataLoaded(player) then
			DataManager.updateStatsToDataStore(player)
		end
	end 
end)
game.Players.PlayerRemoving:Connect(function(player)
	
	if DataManager.playerDataLoaded(player) then
         DataManager.updateStatsToDataStore(player)
	end
	DataManager.removeStats(player)
end) 

When a player joins, when he leaves, when a server shut downs and auto saving.

We have no idea how it happens, when it happens, or how to replicate it, there is no known way to fix it.

It’s used in 3 games currently but only minion simulator has the issue currently which is why we’re so surprised.

https://www.roblox.com/games/4746492207/Minion-Simulator
https://www.roblox.com/games/4168452999/Magic-Sim
https://www.roblox.com/games/3303159734/Shrink-Ray-Simulator-Update-2-SPACE

We’re offering a 10,000 Robux bounty for anyone who is able to fix it (via group funds).

If I had anymore info I could show I would, but I’ve got nothing more to show.

7 Likes

You might’ve hit the 260K character limit?

1 Like

I can’t change out the data saving system while there’s been millions who have joined.

This bug only happens with minion sim currently.

This function retrieves the value of a key from a data store and updates it with a new value. Since this function validates the data, it should be used in favor of SetAsync() when there’s a chance that more than one server can edit the same data at the same time.

There’s no chance that one or more server can update it, it’s not the issue.

Nope, those who had the error were nowhere close to it.

Could you give hint to errors or warnings in the output? It could be an issue with too many requests stacking up

None at all, we’ve debugged everything possible.

We also save every 60 seconds, it would explain maybe 5-10 minutes of failure but not 2-5 hours of them.

I’m trying to figure out if it’s an issue of recieving the data or sending it

That’s the same problem we’ve had, we’re not sure which one it is, and we’ve debugged as much as possible.

You should definitely have some checks to ensure that you’re not overwriting any previous data with nil. This is easily done with :UpdateAsync()

dataStore:UpdateAsync(player.UserId, function(oldValue)
    -- Apply logic here to make sure that the new value makes sense given the old value.
    if Copy then
        return Copy
    else
        return oldValue
    end
end)
1 Like

This may sound stupid, but why not just adapt the game to use datastore2?

2 Likes

In order for their stats to save, they have to be loaded, which we have on a check. (playerDataLoaded)

I’m planning to, I’m willing to give 10,000 nonetheless.

There’s obviously an issue somewhere in that check, and it doesn’t hurt at all to have redundancy when it comes to something as sensitive as player data. You should always be doing your checks as close to the data being set as possible.

If you’d like help migrating to DataStore2, I’d be glad to assist.

2 Likes

I skimmed around the game a bit, and noticed that the leaderboards have some seemingly high values (top person has a quadrillion cash according to your game)
Can you ensure that this data loss is random across all users who have reported it? High number values may cause a data entry to act oddly.

Also, as @AspectW mentioned, you may want to check the size of the data.
You can do print( string.len(game:GetService("HttpService"):JSONEncode(data) ) to test it.

1 Like

This provokes an idea, tostring() all numbers and convert back/save as a JSON table string instead of the array?

Yeah unfortunately I’m a minecraft developer, thought I had to do data saving from scratch didn’t think there was a good working public one.

Still weirds me out nonetheless, I’ll keep the bounty open.

Would you wanna discuss on Discord?

I had that issue once in a game that went above the quintillions, and I used that method to fix it. I don’t believe it’s the best though, not sure

In callbacks passed to DataModel::BindToClose you are only given 30 seconds for all of them to complete. When those 30 seconds are up the server shuts down regardless of them being finished or not. I am not saying 100% of the issue is because of this but maybe this tutorial I wrote a few months ago might help. How to properly save player data in data stores upon server close

The error tends to happen with low currency numbers.

I don’t see quadrillion being a major issue as of right now.