My Datastore module may fail to save player data

Earlier this year, I made a datahub module to centrally call and save the data of the player’s various functional services. I’ve been using it for my first few games, but now one of my games hits 15k ccu and hundreds of players every day report missing data to me (failed to save this game’s data when quitting , but without losing data from previous games)

I am very distressed at the moment. Later, I tried to make a log of the failure to save the data through the webhook of discord, and then wrote the logic of retrying after the failure, and also added the log if the data was successfully saved after the retry. But currently I’m seeing badly that the two log related players don’t match. Then the player reporting the data loss doesn’t appear in either log either


The probability of this kind of problem happening is about 1-3%, so my team and I cannot complete the test. You can only adjust the code of the script each time, publish it to roblox and update the version, and then observe whether the number of players reporting missing data has decreased

Next, I will show you my data storage module, please help me to find out what may be causing the player data storage failure.

DataHubAPIService - MoudleScript
local DataHubAPIService = {}
local DataHubStoreService = require(script.Parent.DataHubStoreService)

function DataHubAPIService.SavePlayerData(Player,DataServiceName,ServiceData)
	local Data = DataHubStoreService.getPlayerDataFormStore(Player)
	Data[DataServiceName] = ServiceData

	DataHubStoreService.UpdatePlayerData(Player,Data)
end

function DataHubAPIService.GetPlayerData(player,DataServiceName)
	local Data = DataHubStoreService.getPlayerDataFormStore(player)
	if Data[DataServiceName] then
		return Data[DataServiceName]
	else
		return nil
	end
end


return DataHubAPIService

DataHubStoreService MoudleScript
local DataHubStoreService = {}
--Cache
local PlayerDataMap = {}
--DataStoreService
local DataStoreService = game:GetService("DataStoreService")
--DataStore
local PlayerDataStore = DataStoreService:GetDataStore("PlayerData")

--errormessage
local DataErrorFeedBack = game:GetService("ReplicatedStorage").Server.Event.FeedBack:WaitForChild("[S-S]DataErrorFeedBack")

--HttpService
local HttpService = game:GetService("HttpService")
--playerService
local Players = game:GetService("Players")


function DataHubStoreService.getPlayerDataFormStore(player)

	if PlayerDataMap[player.UserId] then
		return PlayerDataMap[player.UserId]
	end
	local success,currentData = pcall(function()
		return PlayerDataStore:GetAsync(player.UserId) 	
	end)
	if success then
		if currentData then
			PlayerDataMap[player.UserId] = currentData
			return	currentData
		else
			local Data = {}
			return Data
		end
	else
		local Data = {}
		return Data
	end
end

function DataHubStoreService.UpdatePlayerData (player,Data)

		PlayerDataMap[player.UserId] = Data		
	
end


function DataHubStoreService.SavePlayerDataToDataBase(player)
	
	if PlayerDataMap[player.UserId] then
		local Data = PlayerDataMap[player.UserId]
		
		if not Data then
			return
		end
		
		--OfflineTime
		if PlayerDataMap[player.UserId].PlayerOffline  then
			PlayerDataMap[player.UserId].PlayerOffline.OfflineTime = os.time()
			PlayerDataMap[player.UserId].PlayerOffline.reward = false
		else
			PlayerDataMap[player.UserId]["PlayerOffline"] = {}
			PlayerDataMap[player.UserId]["PlayerOffline"].OfflineTime = os.time()
			PlayerDataMap[player.UserId]["PlayerOffline"].reward = false
		end
	
		--OfflineTime
		local success,errorMessage = pcall(function()	
			PlayerDataStore:SetAsync(player.UserId,Data)
		end)
		if not success then
			DataErrorFeedBack:Fire(player,errorMessage)

			for i = 1, 20, 1 do
				--OfflineTime
				if PlayerDataMap[player.UserId].PlayerOffline  then
					PlayerDataMap[player.UserId].PlayerOffline.OfflineTime = os.time()
					PlayerDataMap[player.UserId].PlayerOffline.reward = false
				else
					PlayerDataMap[player.UserId]["PlayerOffline"] = {}
					PlayerDataMap[player.UserId]["PlayerOffline"].OfflineTime = os.time()
					PlayerDataMap[player.UserId]["PlayerOffline"].reward = false
				end

				local success,errorMessage = pcall(function()	
					PlayerDataStore:SetAsync(player.UserId,Data)
			
				end)
				if not success then
					DataErrorFeedBack:Fire(player,errorMessage)
				else
					PlayerDataMap[player.UserId] = nil
					DataErrorFeedBack:Fire(player,"data-retry-succ")
					return
				end			
			end


			
		else	
			PlayerDataMap[player.UserId] = nil
			return
		end
	end
end



game:BindToClose(function()
	for i,player in pairs(Players:GetPlayers()) do
		task.spawn(function()
	
			if PlayerDataMap[player.UserId] then
				--OfflineTime
				if PlayerDataMap[player.UserId].PlayerOffline  then
					PlayerDataMap[player.UserId].PlayerOffline.OfflineTime = os.time()
					PlayerDataMap[player.UserId].PlayerOffline.reward = false
				else
					PlayerDataMap[player.UserId]["PlayerOffline"] = {}
					PlayerDataMap[player.UserId]["PlayerOffline"].OfflineTime = os.time()
					PlayerDataMap[player.UserId]["PlayerOffline"].reward = false
				end

		--OfflineTime
				local Data = PlayerDataMap[player.UserId]
				local success,errorMessage = pcall(function()	
					PlayerDataStore:SetAsync(player.UserId,Data)

				end)
				if not success then
					DataErrorFeedBack:Fire(player,errorMessage)

					for i = 1, 20, 1 do
						--OfflineTime
						if PlayerDataMap[player.UserId].PlayerOffline  then
							PlayerDataMap[player.UserId].PlayerOffline.OfflineTime = os.time()
							PlayerDataMap[player.UserId].PlayerOffline.reward = false
						else
							PlayerDataMap[player.UserId]["PlayerOffline"] = {}
							PlayerDataMap[player.UserId]["PlayerOffline"].OfflineTime = os.time()
							PlayerDataMap[player.UserId]["PlayerOffline"].reward = false
						end

						local success,errorMessage = pcall(function()	
							PlayerDataStore:SetAsync(player.UserId,Data)
					
						end)
						if not success then
							DataErrorFeedBack:Fire(player,errorMessage)
						else
							PlayerDataMap[player.UserId] = nil
							DataErrorFeedBack:Fire(player,"data-retry-succ")
							return
						end			
					end


				else
					PlayerDataMap[player.UserId] = nil
					return
				end

			end
		end)
	end
end)




Players.PlayerRemoving:Connect(DataHubStoreService.SavePlayerDataToDataBase)

game.ReplicatedStorage.debug.OnServerEvent:Connect(function(player)
	DataHubStoreService.SavePlayerDataToDataBase(player)
end)


return DataHubStoreService

DataHubHandle - Scirpt
local players =  game:GetService("Players")
local DataHubStoreService = require(script.Parent.DataHubStoreService)


while true do
    wait(180)
    for index, player in ipairs(players:GetPlayers()) do
        if game.Players:FindFirstChild(player.Name) then
            DataHubStoreService.SavePlayerDataToDataBase(player)
            wait(10)
        end
    end
end


If anyone can help me with this, our team would love to hire you as a tech support consultant
Thanks again!

You say that your game has hit 15k concurrent users, was this a spike or a gradual rise in users?
If it was a gradual rise, did the errors appear after a certain threshold?

Above 3000 ccu, the problem becomes apparent… as the number of players increases, the number of appearances increases… more and more players are reported

This sounds like a quota/limit issue. Could you go into the most affected game and see if there are any errors that show a script timeout? I’m using this thread as reference: Datastore Problem - HTTP 500 (Internal Server Error)