Dataloss with my custom data module

Help and Feedback Scripting Support
#1

Hi, I am using my own data service module but sometimes there are datalosses. Please can someone tell me what is wrong with my module!

Data Service Module:

local dataService = {}

-- Variables
local runService = game:GetService("RunService")
local dataSS, dataStore
if runService:IsServer() then
    dataSS = game:GetService("DataStoreService")
    dataStore = dataSS:GetDataStore("OfficialDataStoreV1")
end

local http = game:GetService("HttpService")

local playersData = {}

-- Functions

local function retrieveData(plr)
	local data
	local success, err = pcall(function()
		data = dataStore:GetAsync("plr_".. plr.UserId) or nil
	end)
	local count = 0
	while not success do
		warn(err)
		count += 1
		local success, err = pcall(function()
			data = dataStore:GetAsync("plr_".. plr.UserId) or nil
		end)
		if success then
			print("Took ".. count .." time(s) to retrieve data")
		else
			warn(err)
			wait(1)
		end
	end
	return data
end

local function saveData(plr)
	local data = playersData[plr.Name]
	local success, err = pcall(function()
		dataStore:SetAsync("plr_".. plr.UserId, data)
	end)
	local count = 0
	while not success do
		warn(err)
		count += 1
		local success, err = pcall(function()
			dataStore:SetAsync("plr_".. plr.UserId, data)
		end)
		if success then
			print("Took ".. count .." time(s) to save data")
		else
			warn(err)
			wait(1)
		end
	end
end

function dataService.SortData(plr, dataBaseName, func)
	local data = dataService.RetrieveData(plr, dataBaseName)
	if #data == 0 then return end
	table.sort(data,func)
end

function dataService.WaitForData(plr, databaseName)
	if not playersData[plr.Name] then
		repeat wait(0.1) until
		playersData[plr.Name]
	end
	if not playersData[plr.Name].Loaded then
		repeat wait(0.1) until
		playersData[plr.Name].Loaded
	end
	if not playersData[plr.Name][databaseName] then
		playersData[plr.Name][databaseName] = {}
	end
end

local function getFolderWithID(databaseFolder, id)
	for i, v in pairs(databaseFolder:GetChildren()) do
		local idType = "ID"
		if v.PetID then
			idType = "PetID"
		end
		if v[idType].Value == id then
			return v
		end
	end
end

local function getFolderWithName(databaseFolder, dataName)
	for i, v in pairs(databaseFolder:GetChildren()) do
		if v.Name == dataName then
			return v
		end
	end
end


local dataTypes = {
	["boolean"] = "BoolValue",
	["number"] = "IntValue",
	["string"] = "StringValue",
	["Vector3"] = "Vector3Value",
	["Color3"] = "Color3Value",
}

function dataService.SetDataUsingName(plr, databaseName, dataName, propertyName, value)
	local data = dataService.RetrieveData(plr, databaseName)
	for i, v in pairs(data) do
		if v.Name and v.Name == dataName or i == dataName then
			v[propertyName] = value
			if plr:FindFirstChild(databaseName) then
				local dataTemplate = getFolderWithName(plr[databaseName], dataName)
				if not dataTemplate then break end
				if not dataTemplate:FindFirstChild(propertyName) then break end
				for i, v in pairs(dataTypes) do
					if dataTemplate[propertyName]:IsA(v) then
						dataTemplate[propertyName].Value = value
					end
				end
			end
			break
		end
	end
end

function dataService.SetDataUsingID(plr, databaseName, dataID, propertyName, value)
	local data = dataService.RetrieveData(plr, databaseName)
	for i, v in pairs(data) do
		local idType = "ID"
		if v.PetID then
			idType = "PetID"
		end
		if v[idType] and v[idType] == dataID then
			v[propertyName] = value
			if plr:FindFirstChild(databaseName) then
				local dataTemplate = getFolderWithID(plr[databaseName], dataID)
				if not dataTemplate then break end
				dataTemplate[propertyName].Value = value
			end
			break
		end
	end
end

function dataService.TemplateToData(template)
	local newTable = {}
	newTable.Name = template.Name
	for i, v in pairs(template:GetChildren()) do
		newTable[v.Name] = v.Value
	end
	return newTable
end

function dataService.DataToFolder(plr, databaseName)
	dataService.WaitForData(plr, databaseName)

	local folder = Instance.new("Folder")
	folder.Name = databaseName
	folder.Parent = plr

	for i, dataTable in pairs(playersData[plr.Name][databaseName]) do
		local dataTemplateFolder = Instance.new("Folder")
		dataTemplateFolder.Name = dataTable.Name or dataTable.ID or "nil"
		dataTemplateFolder.Parent = folder
		for i, data in pairs(dataTable) do
			local value = Instance.new(dataTypes[typeof(data)])
			value.Name = i  
			value.Value = data
			value.Parent = dataTemplateFolder
		end
	end
	return folder
end

function dataService.OnPlayerJoined(plr)
	if playersData[plr.Name] then return end
	local data = retrieveData(plr)
	if not data then
		print("New Player joined the game")
		playersData[plr.Name] = {Loaded = true}
		-- Player is new to the game if no data has been found
	else
		-- Load the data
		playersData[plr.Name] = data
		playersData[plr.Name].Loaded = true
	end
end

function dataService.SavePlayerData(plr)
	if not playersData[plr.Name] then return end
	saveData(plr)
end

function dataService.OnPlayerLeave(plr)
	if not playersData[plr.Name] then return end
	saveData(plr)
	playersData[plr.Name] = nil
end

function dataService.OnGameClosed()
	for i, plr in pairs(game.Players:GetPlayers()) do
		dataService.OnPlayerLeave(plr)
	end
end

function dataService.RetrieveData(plr, databaseName)
	if not playersData[plr.Name] then
		repeat wait(0.1) until
		playersData[plr.Name]
	end

	if not playersData[plr.Name].Loaded then
		repeat wait(0.1) until
		playersData[plr.Name].Loaded
	end

	if not playersData[plr.Name][databaseName] then
		return nil
	else
		return playersData[plr.Name][databaseName]
	end
end

-- Adds data to a string key
function dataService.SetDataToString(plr, databaseName, dataName, dataTable)
	--print("Adding Data: ", plr, databaseName .." with string: ".. dataName)
	dataService.WaitForData(plr, databaseName)
	playersData[plr.Name][databaseName][dataName] = dataTable
	if typeof(dataTable) == "table" and dataTable.ID then
		return dataTable.ID
	end
end

-- Adds data with a unique identifier
function dataService.AddData(plr, databaseName, dataTable)
	--print("Adding Data: ", plr, databaseName)
	dataService.WaitForData(plr, databaseName)
	if not dataTable.PetID then
		dataTable.ID = http:GenerateGUID()
	end
	table.insert(playersData[plr.Name][databaseName], dataTable)
	return dataTable.ID
end

function dataService.RemoveData(plr, databaseName, dataID)
	if not playersData[plr.Name] or not playersData[plr.Name][databaseName] then return end
	for i, v in pairs(playersData[plr.Name][databaseName]) do
		local idType = "ID"
		if v.PetID then
			idType = "PetID"
		end
		if v[idType] and v[idType] == dataID  then
			table.remove(playersData[plr.Name][databaseName], i)
			break
		end
	end
end

return dataService

Script:

local rs = game:GetService("ReplicatedStorage")
local modules = game:GetService("ServerScriptService"):WaitForChild("Modules")
local dataS = require(modules:WaitForChild("DataService"))
local runS = game:GetService("RunService")
local dataRF = rs:WaitForChild("Events"):WaitForChild("RemoteFunctions"):WaitForChild("Data")

local autoSaveInterval = 180

for i, plr in pairs(game.Players:GetPlayers()) do
	dataS.OnPlayerJoined(plr)
end


game.Players.PlayerAdded:Connect(function(plr)
	dataS.OnPlayerJoined(plr)
end)

game.Players.PlayerRemoving:Connect(function(plr)
	dataS.OnPlayerLeave(plr)
end)

game:BindToClose(function()
	--if runS:IsStudio() then return end
	dataS.OnGameClosed()
end)

dataRF.OnServerInvoke = function(plr, databaseName)
	return dataS.RetrieveData(plr, databaseName)
end

while wait(autoSaveInterval) do
	for i, plr in pairs(game.Players:GetPlayers()) do
		dataS.SavePlayerData(plr)
	end
end
#2

Wrap this in task.spawn like so:

task.spawn(dataService.OnPlayerLeave, plr)
#3

My players still get data loss. I don’t know if it’s the script or what but there is definitely dataloss. Also, it is not because of data limit because i check with #(http:JSONEncode(data)) and it is no where near data limit.

#4

try to save encoded data, then decode it after getting from datastore.

#5

I actually did that right after I made this post but still sometimes there is data loss.
Current Module on writing this:

local dataService = {}

-- Variables
local runService = game:GetService("RunService")
local dataSS, dataStore
if runService:IsServer() then
    dataSS = game:GetService("DataStoreService")
    dataStore = dataSS:GetDataStore("OfficialDataStoreV1")
end

local http = game:GetService("HttpService")
local playersData = {}
local webhook = "https://discord.com/api/webhooks/881909259938443274/7kYh5vNbLpcYB3i5WmxQyhT5FuiGhBt4f2itmnWoPv2SNple7QcYWWCL2jSedJjBPvj9"


-- Functions

local function sendDiscordWebhook(plr, err)
	task.spawn(function()
		if plr ~= nil then
			local success, err = pcall(function()
				local color = Color3.fromRGB(255,0,0)
				local data = http:JSONEncode({
					["embeds"] = {{
						["title"] = "Datastore Error!",
						["description"] = "Player: ".. plr.Name,
						["color"] = (color or 16766976),
						["footer"] = {
							["text"] = err
						},
					}}
				})
				if data ~= nil then
					http:PostAsync(webhook, data)
				end
			end)
			if not success then
				warn(err)
			end
		end
		return
	end)
end

local function retrieveData(plr)
	local data
	local success, err = pcall(function()
		data = dataStore:GetAsync("plr_".. plr.UserId) or nil
	end)
	local count = 0
	while not success do
		warn(err)
		sendDiscordWebhook(plr, err)
		count += 1
		success, err = pcall(function()
			data = dataStore:GetAsync("plr_".. plr.UserId) or nil
		end)
		if success then
			print("Took ".. count .." time(s) to retrieve data")
                        break
		else
			warn(err)
			task.wait(1)
		end
	end
	if typeof(data) == "string" then
		if #data > 200000 then
			sendDiscordWebhook(plr, "Player has ".. #data .." data")
		end
		data = http:JSONDecode(data)		
	end
	return data
end

local function saveData(plr)
	task.spawn(function()
		local data = playersData[plr.Name]
		data = http:JSONEncode(data)
		if #data > 200000 then
			sendDiscordWebhook(plr, "Player has ".. #data .." data")
		end
		local success, err = pcall(function()
			dataStore:SetAsync("plr_".. plr.UserId, data)
		end)
		local count = 0
		while not success do
			warn(err)
			sendDiscordWebhook(plr, err)
			count += 1
			success, err = pcall(function()
				dataStore:SetAsync("plr_".. plr.UserId, data)
			end)
			if success then
				print("Took ".. count .." time(s) to save data")
                                break
			else
				warn(err)
				task.wait(1)
			end
		end
	end)
end

function dataService.SortData(plr, dataBaseName, func)
	local data = dataService.RetrieveData(plr, dataBaseName)
	if #data == 0 then return end
	table.sort(data,func)
end

function dataService.WaitForData(plr, databaseName)
	if not playersData[plr.Name] then
		repeat task.wait(0.1) until
		playersData[plr.Name]
	end
	if not playersData[plr.Name].Loaded then
		repeat task.wait(0.1) until
		playersData[plr.Name].Loaded
	end
	if not playersData[plr.Name][databaseName] then
		playersData[plr.Name][databaseName] = {}
	end
end

local function getFolderWithID(databaseFolder, id)
	for i, v in pairs(databaseFolder:GetChildren()) do
		local idType = "ID"
		if v.PetID then
			idType = "PetID"
		end
		if v[idType].Value == id then
			return v
		end
	end
end

local function getFolderWithName(databaseFolder, dataName)
	for i, v in pairs(databaseFolder:GetChildren()) do
		if v.Name == dataName then
			return v
		end
	end
end


local dataTypes = {
	["boolean"] = "BoolValue",
	["number"] = "IntValue",
	["string"] = "StringValue",
	["Vector3"] = "Vector3Value",
	["Color3"] = "Color3Value",
}

function dataService.SetDataUsingName(plr, databaseName, dataName, propertyName, value)
	local data = dataService.RetrieveData(plr, databaseName)
	for i, v in pairs(data) do
		if v.Name and v.Name == dataName or i == dataName then
			v[propertyName] = value
			if plr:FindFirstChild(databaseName) then
				local dataTemplate = getFolderWithName(plr[databaseName], dataName)
				if not dataTemplate then break end
				if not dataTemplate:FindFirstChild(propertyName) then break end
				for i, v in pairs(dataTypes) do
					if dataTemplate[propertyName]:IsA(v) then
						dataTemplate[propertyName].Value = value
					end
				end
			end
			break
		end
	end
end

function dataService.SetDataUsingID(plr, databaseName, dataID, propertyName, value)
	local data = dataService.RetrieveData(plr, databaseName)
	for i, v in pairs(data) do
		local idType = "ID"
		if v.PetID then
			idType = "PetID"
		end
		if v[idType] and v[idType] == dataID then
			v[propertyName] = value
			if plr:FindFirstChild(databaseName) then
				local dataTemplate = getFolderWithID(plr[databaseName], dataID)
				if not dataTemplate then break end
				dataTemplate[propertyName].Value = value
			end
			break
		end
	end
end

function dataService.TemplateToData(template)
	local newTable = {}
	newTable.Name = template.Name
	for i, v in pairs(template:GetChildren()) do
		newTable[v.Name] = v.Value
	end
	return newTable
end

function dataService.DataToFolder(plr, databaseName)
	dataService.WaitForData(plr, databaseName)

	local folder = Instance.new("Folder")
	folder.Name = databaseName
	folder.Parent = plr

	for i, dataTable in pairs(playersData[plr.Name][databaseName]) do
		local dataTemplateFolder = Instance.new("Folder")
		dataTemplateFolder.Name = dataTable.Name or dataTable.ID or "nil"
		dataTemplateFolder.Parent = folder
		for i, data in pairs(dataTable) do
			local value = Instance.new(dataTypes[typeof(data)])
			value.Name = i  
			value.Value = data
			value.Parent = dataTemplateFolder
		end
	end
	return folder
end

function dataService.OnPlayerJoined(plr)
	if playersData[plr.Name] then return end
	local data = retrieveData(plr)
	if not data then
		playersData[plr.Name] = {Loaded = true}
		-- Player is new to the game if no data has been found
	else
		-- Load the data
		playersData[plr.Name] = data
		playersData[plr.Name].Loaded = true
	end
end

function dataService.SavePlayerData(plr)
	if not playersData[plr.Name] then return end
	saveData(plr)
end

function dataService.OnPlayerLeave(plr)
	if not playersData[plr.Name] then return end
	saveData(plr)
	playersData[plr.Name] = nil
end

function dataService.OnGameClosed()
	for i, plr in pairs(game.Players:GetPlayers()) do
		dataService.OnPlayerLeave(plr)
	end
end

function dataService.RetrieveData(plr, databaseName)
	if not playersData[plr.Name] then
		repeat task.wait(0.1) until
		playersData[plr.Name]
	end

	if not playersData[plr.Name].Loaded then
		repeat task.wait(0.1) until
		playersData[plr.Name].Loaded
	end

	if not playersData[plr.Name][databaseName] then
		return nil
	else
		return playersData[plr.Name][databaseName]
	end
end

-- Adds data to a string key
function dataService.SetDataToString(plr, databaseName, dataName, dataTable)
	--print("Adding Data: ", plr, databaseName .." with string: ".. dataName)
	dataService.WaitForData(plr, databaseName)
	playersData[plr.Name][databaseName][dataName] = dataTable
	if typeof(dataTable) == "table" and dataTable.ID then
		return dataTable.ID
	end
end

-- Adds data with a unique identifier
function dataService.AddData(plr, databaseName, dataTable)
	--print("Adding Data: ", plr, databaseName)
	dataService.WaitForData(plr, databaseName)
	if not dataTable.PetID then
		dataTable.ID = http:GenerateGUID()
	end
	table.insert(playersData[plr.Name][databaseName], dataTable)
	return dataTable.ID
end

function dataService.RemoveData(plr, databaseName, dataID)
	if not playersData[plr.Name] or not playersData[plr.Name][databaseName] then return end
	for i, v in pairs(playersData[plr.Name][databaseName]) do
		local idType = "ID"
		if v.PetID then
			idType = "PetID"
		end
		if v[idType] and v[idType] == dataID then
			table.remove(playersData[plr.Name][databaseName], i)
			break
		end
	end
end

return dataService
#6

Try using UpdateAsync. Also, I believe there are Datastore V2 functions that allow for versioning, allowing you to revert data to a previous version, but that’s a little more complicated.