Checking through Data Table for any missing values

Help and Feedback Scripting Support
#1

So I’m making a custom Data store, and I want to go through all the tables to check if there is any missing values in said table compared to the template. I’ve tried to do so but the “missing values” insert in every table.

I want to know how to insert values into tables without having to completely wipe everyone’s data.
Also remove any values that shouldn’t exist anymore, So I can efficiently remove that save from everyone’s data saves without, of course, wiping everything.

Data Code

local Players = game:GetService("Players")
local RunService = game:GetService("RunService")
local DataStoreService = game:GetService("DataStoreService")

local DataVersion = script:GetAttribute("DataVersion")
local DataStore = DataStoreService:GetDataStore("PlayerStore" .. tostring(DataVersion))
local KeyPrefix = "PlayerData-"

local Loaded = {}


local data = {}
data.__index = data

function data.new(player)
	local self = setmetatable({}, data)
	
	self.PlayerData = {
		Weapon = {Name = "ThermalKatana", Type = "Sword"},
		Name = {First = "N/A", IsLeader = false},
		Clan = "N/A",
		Race = {Name = "Human", Type = "Normal"},
		Rank = 0,
		Reputation = 0,
		Doubloons = 0,
		
		Inventory = {
			Moves = {},
			Armor = {},
			Items = {}
		},
		
		Attributes = {},
		DamageInfo = {},
		CharacterInfo = {},
		
		ImAFreak = "Yes",
		
		BaseWalkspeed = 16,
		BaseJumppower = 25,
	}
	self.ModerationData = {
		IsBanned = false,
		BanReason = "You shouldn't be banned right now, report.",
		BanUser = "Rymxi"
	}
	self.InternalData = {
		Keybinds = nil,
		ClientSettings = nil
	}
	
	return self
end

function data.GetData(Player:Player)
	if not Loaded[Player] then
		return;
	end

	return Loaded[Player]
end

function data.Load(Player:Player)
	if Loaded[Player] then
		return;
	end

	local Key = KeyPrefix .. tostring(Player.UserId)
	local Success, Error
	local Data

	repeat
		Success, Error = pcall(function()
			Data = DataStore:GetAsync(Key)
		end)
	until Success

	if not Success then Player:Kick("Failed to load data" ..  tostring(Error)) return end;

	if not Data then print("No data found, creating new save") Data = data.new(Player) end;

	for _,Table in pairs(Data) do
		task.spawn(function()
			
		end)
	end

	
	Loaded[Player] = Data
	
	print(Data)
	
	return true
end

function  data.Save(Player:Player)

	local Data = data.GetData(Player)

	local Key = KeyPrefix .. tostring(Player.UserId)
	local Success, Error

	if not Data then return end;

	repeat
		Success, Error = pcall(function()
			Data = DataStore:SetAsync(Key, Data)
		end)
	until Success

	if not Success then warn("Didn't save:" .. tostring(Error)) return false end;

	Loaded[Player] = Data
	return true
end

function data.OnGameClose()
	if RunService:IsStudio() then
		return;
	end
	
	local PlayersLeft =  #Players:GetPlayers()
	
	while PlayersLeft > 0 do
		task.spawn(function()
			for _,Player in Players:GetPlayers() do
				data.Save(Player)
				
				PlayersLeft -= 1
			end
		end)
	end
end

function data.Init()
	print("Data Init")
	
	Players.PlayerAdded:Connect(function(Player)
		data.Load(Player)
	end)
	
	Players.PlayerRemoving:Connect(function(Player)
		data.Save(Player)
		Loaded[Player] = nil
	end)
end

return data

the for loop is the shell of a previous attempt to check stuff

1 Like
#2

I’m assuming you want to add reconciling for your data. You can check if there is a valid index with the dictionary in a loop and add data if there isn’t.

Code:

-- Assuming self.PlayerData is your template and Data is the player's current data
for key, value in self.PlayerData do
	if not Data[key] then
		Data[key] = value
	end
end

This will only shallow reconcile (as in only copy values that are direct children of the table)

#3

The method is usually called “reconciling” in programming, and it is as simple as filling in missing values:

for key, value in template do
    if original[key] == nil then
        original[key] = value
    end
end

Since we are dealing with nested tables here, we would need to perform a deep reconciliation via recursion making sure we clone values that are tables for a different memory address, or else changing a data value will also change the template value.

Code 
-- performs a deep table clone operation
local function clone_deep(t)
	local cloned = {}
	
	for key, value in t do
		if typeof(value) == "table" then
			cloned[key] = clone_deep(value)
		else
			cloned[key] = value
		end
	end
	
	return cloned
end

local function reconcile(original, template)
	-- iterate through our template key value pairs
	for key, defaultValue in template do
		-- check if our original data does not contain 'key'
		if original[key] == nil then
			if typeof(defaultValue) == "table" then
				-- if the 'defaultValue' is a table, we assign a deep clone of it
				original[key] = clone_deep(defaultValue)
			else
				-- otherwise, we assign as is
				original[key] = defaultValue
			end
		end
	end
end
Example 
local Template = {
	Level = 1,
	Experience = 0,
	Gold = 100,
        VIP = false,
	Inventory = {
		Items = {},
		Equipped = {
			Primary = "",
			Secondary = ""
		}
	}
}

local LoadedData = {
	Level = 50,
	Experience = 777
}

reconcile(LoadedData, Template)
print("Reconciled data:", LoadedData)

--[=[
	Reconciled data:  ▼  {
        ["Experience"] = 777,
        ["Gold"] = 100,
        ["Inventory"] =  ▼  {
           ["Equipped"] =  ▼  {
              ["Primary"] = "",
              ["Secondary"] = ""
           },
           ["Items"] = {}
        },
        ["Level"] = 50,
        ["VIP"] = false
    }
]=]

Keep in mind: from the example template above, what if we change the “VIP” key to be a number, 0 being not VIP and higher number = higher status instead of being a boolean. Now you run into the problem where the old data will contain key value pairs of the wrong type and can bring runtime errors. To account for that, it is often a good idea to simply check if the types are not the same and to replace it.

Code 
local function clone_deep(t)
	local cloned = {}

	for key, value in t do
		if typeof(value) == "table" then
			cloned[key] = clone_deep(value)
		else
			cloned[key] = value
		end
	end

	return cloned
end

local function reconcile(original, template)
	for key, defaultValue in template do
		--if original[key] == nil then
		if typeof(original[key]) ~= typeof(defaultValue) then
			if typeof(defaultValue) == "table" then
				original[key] = clone_deep(defaultValue)
			else
				original[key] = defaultValue
			end
		end
	end
end
Example 
local Template = {
	VIP = 0
}

local LoadedData = {
	VIP = true
}

reconcile(LoadedData, Template)
print("Reconciled data:", LoadedData)

--[=[
	Reconciled data:  ▼  {
        ["VIP"] = 0
    }
]=]

Automatically removing keys are, in my opinion, very dangerous and can lead to the loss of data if not handled perfectly. People tend to just leave it in an event that a removed feature might get reinstated in the future or manually remove keys.

You might think the solution is as simple as “oh, let’s simply check the template if it doesn’t contain the key”, and while this is on the right track, can lead to some problems.

Code 
local function clone_deep(t)
	local cloned = {}

	for key, value in t do
		if typeof(value) == "table" then
			cloned[key] = clone_deep(value)
		else
			cloned[key] = value
		end
	end

	return cloned
end

-- removes missing keys with a deep operation
local function removeMissingKeys(original, template)
	for key in original do
		if template[key] == nil then
			original[key] = nil
		elseif typeof(original[key]) == "table" and typeof(template[key]) == "table" then
			removeMissingKeys(original[key], template[key])
		end
	end
end

local function reconcile(original, template)
	-- matching values
	for key, defaultValue in template do
		if typeof(original[key]) ~= typeof(defaultValue) then
			if typeof(defaultValue) == "table" then
				original[key] = clone_deep(defaultValue)
			else
				original[key] = defaultValue
			end
		end
	end

	-- removing missing keys
	removeMissingKeys(original, template)
end
Example 
local Template = {
	Level = 1,
	Experience = 0,
	Gold = 0,
	Equipped = {
		Primary = "",
		Secondary = ""
	}
}

local LoadedData = {
	Level = 10,
	Experience = 100,
	Cash = 1000,
	Equipped = {
		Primary = "",
		Secondary = "",
		Pet = ""
	}
}

reconcile(LoadedData, Template)
print("Reconciled data:", LoadedData)

--[=[
	Reconciled data:  ▼  {
        ["Equipped"] =  ▼  {
           ["Primary"] = "",
           ["Secondary"] = ""
        },
        ["Experience"] = 100,
        ["Gold"] = 0,
        ["Level"] = 10
    }
]=]

This may look fine. However, let’s introduce a new Items table containing a table data of their items.

Example 
local Template = {
	Level = 1,
	Experience = 0,
	Gold = 0,
	Equipped = {
		Primary = "",
		Secondary = "",
		Items = {}
	}
}

local LoadedData = {
	Level = 10,
	Experience = 100,
	Cash = 1000,
	Equipped = {
		Primary = "",
		Secondary = "",
		Pet = "",
		Items = {
			Sword = 1,
			Apple = 64
		}
	}
}

reconcile(LoadedData, Template)
print("Reconciled data:", LoadedData)

--[=[
	Reconciled data:  ▼  {
	    ["Equipped"] =  ▼  {
	       ["Items"] = {},
	       ["Primary"] = "",
	       ["Secondary"] = ""
	    },
	    ["Experience"] = 100,
	    ["Gold"] = 0,
	    ["Level"] = 10
	}
]=]

Running the reconcile function completely wiped their items since in the original template, “Sword” and “Apple” does not exist. A way to fix this is to let the code assume (and for you, the developer, to follow) the rule that empty template tables and it’s subsequent key value pairs will get ignored. Do keep in mind that this is not a one size fits all solution, as not all data tables will want to get treated under this assumption.

Code 
local function clone_deep(t)
	local cloned = {}

	for key, value in t do
		if typeof(value) == "table" then
			cloned[key] = clone_deep(value)
		else
			cloned[key] = value
		end
	end

	return cloned
end

local function removeMissingKeys(original, template)
	if #template > 0 then
		for key in original do
			if template[key] == nil then 
				original[key] = nil
			elseif typeof(original[key]) == "table" and typeof(template[key]) == "table" then
				removeMissingKeys(original[key], template[key])
			end
		end
	end
end

local function reconcile(original, template)
	for key, defaultValue in template do
		if typeof(original[key]) ~= typeof(defaultValue) then
			if typeof(defaultValue) == "table" then
				original[key] = clone_deep(defaultValue)
			else
				original[key] = defaultValue
			end
		end
	end

	removeMissingKeys(original, template)
end
Example 
local Template = {
	Level = 1,
	Experience = 0,
	Gold = 0,
	Equipped = {
		Primary = "",
		Secondary = "",
		Items = {}
	}
}

local LoadedData = {
	Level = 10,
	Experience = 100,
	Cash = 1000,
	Equipped = {
		Primary = "",
		Secondary = "",
		Pet = "",
		Items = {
			Sword = 1,
			Apple = 64
		}
	}
}

reconcile(LoadedData, Template)
print("Reconciled data:", LoadedData)

--[=[
	Reconciled data:  ▼  {
	    ["Cash"] = 1000,
	    ["Equipped"] =  ▼  {
	       ["Items"] =  ▼  {
	          ["Apple"] = 64,
	          ["Sword"] = 1
	       },
	       ["Pet"] = "",
	       ["Primary"] = "",
	       ["Secondary"] = ""
	    },
	    ["Experience"] = 100,
	    ["Gold"] = 0,
	    ["Level"] = 10
	}
]=]

Edited for readability for future readers

1 Like
#4

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.