DataPredict [Release 1.21] - General Purpose Machine Learning And Deep Learning Library (Learning AIs, Generative AIs, and more!)

Resources Community Resources
, ,
#651

Oh, the genetic algorithm UI was for my old implementation, I repurposed it to just keep track of how long the car has been living for

Edit: did you update MatrixL?

local sStorage    = game:GetService('ServerStorage')
local repStorage  = game:GetService('ReplicatedStorage')
local DataStore   = game:GetService('DataStoreService')

local Gizmo = require(repStorage.Gizmo)

workspace:SetAttribute("GizmosEnabled", true)

local ActorModelStorage = DataStore:GetDataStore('ActorModelStorage')
local CriticModelStorage = DataStore:GetDataStore('CriticModelStorage')
local DataPredict = require(repStorage.DataPredict)
local InputDataEvent = script.InputData
local SaveParamsEvent = script.Save

local NPCHumanoids = workspace.NPCHumanoids
local PathfinderCourse = workspace.PathfinderCourse

local CityMap = workspace.CityMap
local Car   = sStorage["2014 Chevy Caprice PPV"]
local Start = CityMap.Start
local Goal  = CityMap.Goal

local raycastParams = RaycastParams.new()
raycastParams.FilterType = Enum.RaycastFilterType.Exclude
raycastParams.FilterDescendantsInstances = {Car}

local RayParts = Car.RayParts

--print('Saved Model Parameters: ', ModelStorage:GetAsync(script.StorageID.Value))

local MatrixL =  require(repStorage.DataPredict.MatrixL)

local maxRewardArrayLength = 100

local maxCurrentArrayLength = 100

local isRewardedArray = {}

local currentAccuracyArray = {}

local classesList = {'IncreaseThrottle', 'IncreaseLeftSteer', 'DecreaseThrottle', 'IncreaseRightSteer'}

local Optimizer = DataPredict.Optimizers.AdaptiveMomentEstimation.new()
 
local Reg = DataPredict.Others.Regularization.new()


local Attempts = 0
local Controller = require(Car.Controls)


--[[local function buildNPC()
	
	local npcClone = NPC:Clone()
	npcClone.Parent=NPCHumanoids
	npcClone:PivotTo(Start.CFrame)
	
	Controller = require(npcClone.Controls)
	
	return npcClone
	
end]]

local function buildActor()

	local Actor = DataPredict.Models.NeuralNetwork.new(1)

	if not script.LoadModel.Value then
		Actor:setModelParametersInitializationMode("LeCunUniform")

		Actor:addLayer(9, true, 'ReLU', 0.001)

		Actor:addLayer(6, true, 'ReLU', 0.001)

		Actor:addLayer(4, false, 'StableSoftmax', 0.001)
		Actor:setClassesList(classesList)
	else
		print('Loading Actor...')
		Actor:setModelParameters(ActorModelStorage:GetAsync(script.StorageID.Value))
	end

	if script.LoadModel.Value==true then
		Actor:setModelParameters(ActorModelStorage:GetAsync(script.StorageID.Value))
	end

	SaveParamsEvent.Event:Connect(function()
		local ModelParams = Actor:getModelParameters()
		ActorModelStorage:SetAsync(script.StorageID.Value, ModelParams)
	end)

	return Actor

end

local function buildCritic()

	local Critic = DataPredict.Models.NeuralNetwork.new(1)

	if not script.LoadModel.Value then
		Critic:setModelParametersInitializationMode("LeCunUniform")

		Critic:addLayer(9, true, 'ReLU', 0.001)

		Critic:addLayer(4, true, 'ReLU', 0.001)

		Critic:addLayer(1, false, 'Sigmoid', 0.001)

		Critic:setClassesList({1,2,3,4})
	else
		print('Loading Critic...')
		Critic:setModelParameters(CriticModelStorage:GetAsync(script.StorageID.Value))
	end	
	
	if script.LoadModel.Value==true then
		Critic:setModelParameters(CriticModelStorage:GetAsync(script.StorageID.Value))
	end
	
	SaveParamsEvent.Event:Connect(function()
		local ModelParams = Critic:getModelParameters()
		CriticModelStorage:SetAsync(script.StorageID.Value, ModelParams)
	end)

	return Critic

end

local function buildModel()

	local ExperienceReplay = DataPredict.ExperienceReplays.UniformExperienceReplay.new(1, nil, 30)

	local NeuralNetwork = DataPredict.Models.NeuralNetwork.new(1)
	NeuralNetwork:addLayer(9, true, 'ReLU')
	NeuralNetwork:addLayer(6, false, 'ReLU')
	NeuralNetwork:addLayer(#classesList, false, 'StableSoftmax')

	local Model = DataPredict.Models.DeepDoubleQLearningV2.new(1, 0.6)
	local StorageID = script.StorageID.Value
	
	--[[Model:setActorModel(buildActor())
	Model:setCriticModel(buildCritic())]]
	Model:setModel(NeuralNetwork)
	
	local QLearningNeuralNetworkQuickSetup = DataPredict.Others.ReinforcementLearningQuickSetup.new(60,0,0)
	
	--QLearningNeuralNetworkQuickSetup:setExperienceReplay(ExperienceReplay)
	
	QLearningNeuralNetworkQuickSetup:setModel(Model)

	QLearningNeuralNetworkQuickSetup:setClassesList(classesList)
	
	
	--Model:setPrintReinforcementOutput(false)

	return QLearningNeuralNetworkQuickSetup

end

local function buildCar()
	local BuiltCar = Car:Clone()
	BuiltCar.Parent=workspace
	BuiltCar:PivotTo(CityMap.Start.CFrame)
	return BuiltCar
end

local function generateEnvironmentFeatureVector()

	local featureVector1 = MatrixL:createRandomNormalMatrix(1, 3)

	local featureVector2 = MatrixL:createRandomNormalMatrix(1, 3)

	local environmentFeatureVector = MatrixL:subtract(featureVector1, featureVector2)

	environmentFeatureVector[1][1] = 1 -- 1 at first column for bias.

	return environmentFeatureVector

end

local function countTrueBooleansInBooleanArray(booleanArray)

	local numberOfTrueBooleans = 0

	for i, boolean in ipairs(booleanArray) do

		if (boolean == true) then

			numberOfTrueBooleans += 1

		end

	end

	return numberOfTrueBooleans

end

local function calculateCurrentAccuracy(booleanArray)

	local numberOfBooleans = #booleanArray

	local numberOfTrueBooleans = countTrueBooleansInBooleanArray(booleanArray) 

	local currentAccuracy = (numberOfTrueBooleans / numberOfBooleans) * 100

	currentAccuracy = math.floor(currentAccuracy)

	return currentAccuracy

end

local function calculateAverage(array)

	local sum = 0

	local average

	for i, number in ipairs(array) do

		sum += number

	end

	average = (sum / #array)

	return average

end

local function getCurrentAverageAccuracy()

	local currentAccuracy

	local currentAverageAccuracy

	if (#isRewardedArray > maxRewardArrayLength) then

		table.remove(isRewardedArray, 1)

		currentAccuracy = calculateCurrentAccuracy(isRewardedArray)

		table.insert(currentAccuracyArray, currentAccuracy)

	end

	if (#currentAccuracyArray > maxCurrentArrayLength) then

		table.remove(currentAccuracyArray, 1)

		currentAverageAccuracy = calculateAverage(currentAccuracyArray)

	end

	return currentAverageAccuracy

end

-- Function to calculate the angle between two vectors
local function calculateAngle(vector1, vector2)
	local dotProduct = vector1:Dot(vector2)
	local magnitudeProduct = vector1.Magnitude * vector2.Magnitude
	local cosineOfAngle = dotProduct / magnitudeProduct
	local angle = math.acos(cosineOfAngle)
	return math.deg(angle) -- Convert from radians to degrees
end

-- Function to calculate the required rotation for the NPC to face the target part
local function getTurnAngle(BasePart, targetPart)
	local npcPosition = BasePart.Position
	local targetPosition = targetPart.Position

	-- Calculate the direction vector from the NPC to the target part
	local directionToTarget = (targetPosition - npcPosition).unit

	-- Get the NPC's current forward direction (assuming NPC is oriented along the Z-axis)
	local npcForward = BasePart.CFrame.LookVector

	-- Calculate the angle between the NPC's forward direction and the direction to the target
	local angle = calculateAngle(npcForward, directionToTarget)

	-- Determine if the target is to the left or right of the NPC
	local crossProduct = npcForward:Cross(directionToTarget)
	if crossProduct.Y < 0 then
		angle = -angle -- If the cross product's Y component is negative, the target is to the left
	end

	return angle
end


function round(number, decimals)
	number = number*10^decimals
	number = math.floor(number+0.5)
	number = number / 10^decimals
	
	return number
end	

local function startEnvironment(Model, CarModel)

	Controller=require(CarModel.Controls)

	--Model:getModel():clearModelParameters()
	Attempts+=1
	game.StarterGui.GeneticAlgoDebug.Frame.DebugFrame.Attempts.Text = tostring('Attempt, '..Attempts)
	local timeAlive = 0
	game.StarterGui.GeneticAlgoDebug.Frame.DebugFrame.TimeAlive.Text = tostring('Time Alive:, '..math.round(timeAlive))

	local reward = 0
	local defaultReward = 0.01
	local defaultPunishment = -1

	local predictedLabel

	local environmentVector = generateEnvironmentFeatureVector()
	
	local lastRotationErrorY
	local currRotationErrorY= getTurnAngle(CarModel.DriveSeat, Goal)
	
	local FLRayHit = workspace:Raycast(CarModel.RayParts.FL.Position, CarModel.RayParts.FL.CFrame.LookVector*1000, raycastParams)
	local FRRayHit = workspace:Raycast(CarModel.RayParts.FR.Position, CarModel.RayParts.FR.CFrame.LookVector*1000, raycastParams)
	local FRayHit = workspace:Raycast(CarModel.RayParts.F.Position, CarModel.RayParts.F.CFrame.LookVector*1000, raycastParams)
	local LRayHit = workspace:Raycast(CarModel.RayParts.L.Position, CarModel.RayParts.L.CFrame.LookVector*1000, raycastParams)
	local RiRayHit = workspace:Raycast(CarModel.RayParts.Ri.Position, CarModel.RayParts.Ri.CFrame.LookVector*1000, raycastParams)
	local RLRayHit = workspace:Raycast(CarModel.RayParts.RL.Position, CarModel.RayParts.RL.CFrame.LookVector*1000, raycastParams)
	local RRRayHit = workspace:Raycast(CarModel.RayParts.RR.Position, CarModel.RayParts.RR.CFrame.LookVector*1000, raycastParams)
	local RRayHit = workspace:Raycast(CarModel.RayParts.R.Position, CarModel.RayParts.R.CFrame.LookVector*1000, raycastParams)
	
	local FLDist
	local FRDist
	local LDist
	local RiDist
	local FDist
	local RLDist
	local RRDist
	local RDist
	
	local previousPosition
	local currentPosition = CarModel:GetPivot().Position
	
	local previousinverseGoalDistance
	local inverseGoalDistance
	
	local touched = false
	
	if FLRayHit then FLDist=1/FLRayHit.Distance else FLDist=0 end 
	if FRRayHit then FRDist=1/FRRayHit.Distance else FRDist=0 end
	if FRayHit then FDist=1/FRayHit.Distance else FDist=0 end
	if LRayHit then LDist=1/LRayHit.Distance else LDist=0 end
	if RiRayHit then RiDist=1/RRayHit.Distance else RiDist=0 end
	if RLRayHit then RLDist=1/RLRayHit.Distance else RLDist=0 end 
	if RRRayHit then RRDist=1/RRRayHit.Distance else RRDist=0 end
	if RRayHit then RDist=1/RRayHit.Distance else RDist=0 end
	
	
	local function getRewardValue()

		--[[local distanceAdjustmentFactor = 0.1
		local rotationAdjustmentFactor = 0.01

		local distanceChangeReward = (lastDistancefromGoal - currDistancefromGoal) --* distanceAdjustmentFactor
		print('DistanceReward:', distanceChangeReward)

		local rotationChangeReward = 0 --(lastRotationErrorY - currRotationErrorY) --* rotationAdjustmentFactor
		print('RotationReward:', rotationChangeReward)

		local rewardValue = distanceChangeReward + rotationChangeReward]]
		
		local rewardValue = 0 --(currDistancefromGoal < lastDistancefromGoal) or (currDistancefromGoal < 4 and currDistancefromGoal == lastDistancefromGoal)

		return rewardValue

	end
	
	local goalReached=0
	CarModel.Hitbox.Touched:Connect(function(otherPart)
		if otherPart.Name=='Goal' then
			print('Goal Reached!')
			goalReached=1
		elseif not otherPart:IsDescendantOf(CarModel) then	
			print(otherPart, 'Touched!')
			touched=true
		end	
	end)
	
	CarModel.Hitbox.TouchEnded:Connect(function(otherPart)
		if otherPart.Name=='Goal' then
			print('Goal Overshot!!')
			goalReached=0
		end
	end)
	
	local totalReward = 10

	game:GetService('RunService'):BindToRenderStep("DrawGizmos", 1, function ()
		Gizmo.PushProperty('Color3', Color3.new(1, 0.160784, 0.176471))
		if FLRayHit then Gizmo.Ray:Draw(CarModel.RayParts.FL.Position, FLRayHit.Position) end
		if FRRayHit then Gizmo.Ray:Draw(CarModel.RayParts.FR.Position, FRRayHit.Position) end
		if FRayHit then Gizmo.Ray:Draw(CarModel.RayParts.F.Position, FRayHit.Position) end
		if LRayHit then Gizmo.Ray:Draw(CarModel.RayParts.L.Position, LRayHit.Position) end
		if RiRayHit then Gizmo.Ray:Draw(CarModel.RayParts.Ri.Position, RiRayHit.Position) end
		if RLRayHit then Gizmo.Ray:Draw(CarModel.RayParts.RL.Position, RLRayHit.Position) end
		if RRRayHit then Gizmo.Ray:Draw(CarModel.RayParts.RR.Position, RRRayHit.Position) end
		if RRayHit then Gizmo.Ray:Draw(CarModel.RayParts.R.Position, RRayHit.Position) end
		Gizmo.PushProperty('Color3', Color3.new(1, 0.576471, 0.152941))
		if FLRayHit then Gizmo.Sphere:Draw(CFrame.new(FLRayHit.Position), 1, 20, 360) end
		if FRRayHit then Gizmo.Sphere:Draw(CFrame.new(FRRayHit.Position), 1, 20, 360) end
		if FRayHit then Gizmo.Sphere:Draw(CFrame.new(FRayHit.Position), 1, 20, 360) end
		if LRayHit then Gizmo.Sphere:Draw(CFrame.new(LRayHit.Position), 1, 20, 360) end
		if RiRayHit then Gizmo.Sphere:Draw(CFrame.new(RiRayHit.Position), 1, 20, 360) end
		if RLRayHit then Gizmo.Sphere:Draw(CFrame.new(RLRayHit.Position), 1, 20, 360) end
		if RRRayHit then Gizmo.Sphere:Draw(CFrame.new(RRRayHit.Position), 1, 20, 360) end
		if RRayHit then Gizmo.Sphere:Draw(CFrame.new(RRayHit.Position), 1, 20, 360) end
		Gizmo.PushProperty('Color3', Color3.new(0.8, 1, 0.14902))
		if FLRayHit then Gizmo.Text:Draw(CarModel.RayParts.FL.Position, round(FLDist, 2)) end
		if FRRayHit then Gizmo.Text:Draw(CarModel.RayParts.FR.Position, round(FRDist, 2)) end
		if FRayHit then Gizmo.Text:Draw(CarModel.RayParts.F.Position, round(FDist, 2)) end
		if LRayHit then Gizmo.Text:Draw(CarModel.RayParts.L.Position, round(LDist, 2)) end
		if RiRayHit then Gizmo.Text:Draw(CarModel.RayParts.Ri.Position, round(RiDist, 2)) end
		if RLRayHit then Gizmo.Text:Draw(CarModel.RayParts.RL.Position, round(RLDist, 2)) end
		if RRRayHit then Gizmo.Text:Draw(CarModel.RayParts.RR.Position, round(RRDist, 2)) end
		if RRayHit then Gizmo.Text:Draw(CarModel.RayParts.R.Position, round(RDist, 2)) end
		Gizmo.PushProperty('Color3', Color3.new(1, 0, 0.0156863))
		if RRayHit then Gizmo.Text:Draw(CarModel.DriveSeat.Position, CarModel.DriveSeat.AssemblyLinearVelocity.Magnitude) end
	end)

	Controller['Ignition'](true)

	while task.wait(0.01) do
		
		previousPosition=currentPosition
		currentPosition=CarModel:GetPivot().Position
		
		timeAlive+=0.1
		game.StarterGui.GeneticAlgoDebug.Frame.DebugFrame.TimeAlive.Text = tostring('Time Alive:, '..math.round(timeAlive))

		FLRayHit = workspace:Raycast(CarModel.RayParts.FL.Position, CarModel.RayParts.FL.CFrame.LookVector*1000, raycastParams)
		FRRayHit = workspace:Raycast(CarModel.RayParts.FR.Position, CarModel.RayParts.FR.CFrame.LookVector*1000, raycastParams)
		FRayHit = workspace:Raycast(CarModel.RayParts.F.Position, CarModel.RayParts.F.CFrame.LookVector*1000, raycastParams)
		LRayHit = workspace:Raycast(CarModel.RayParts.L.Position, CarModel.RayParts.L.CFrame.LookVector*1000, raycastParams)
		RiRayHit = workspace:Raycast(CarModel.RayParts.Ri.Position, CarModel.RayParts.Ri.CFrame.LookVector*1000, raycastParams)
		RLRayHit = workspace:Raycast(CarModel.RayParts.RL.Position, CarModel.RayParts.RL.CFrame.LookVector*1000, raycastParams)
		RRRayHit = workspace:Raycast(CarModel.RayParts.RR.Position, CarModel.RayParts.RR.CFrame.LookVector*1000, raycastParams)
		RRayHit = workspace:Raycast(CarModel.RayParts.R.Position, CarModel.RayParts.R.CFrame.LookVector*1000, raycastParams)
		
		currRotationErrorY   = getTurnAngle(CarModel.DriveSeat, Goal)
		local DistancefromGoal = (Goal.Position - CarModel:GetPivot().Position).Magnitude
		
		previousinverseGoalDistance = inverseGoalDistance
		inverseGoalDistance = DistancefromGoal>0 and 1/DistancefromGoal or 1

		local OldLDist = LDist
		local OldRDist = RDist

		if FLRayHit then FLDist=FLRayHit.Distance else FLDist=0 end 
		if FRRayHit then FRDist=FRRayHit.Distance else FRDist=0 end
		if FRayHit then FDist=FRayHit.Distance else FDist=0 end
		if LRayHit then LDist=LRayHit.Distance else LDist=0 end
		if RiRayHit then RiDist=RiRayHit.Distance else RiDist=0 end
		if RLRayHit then RLDist=RLRayHit.Distance else RLDist=0 end 
		if RRRayHit then RRDist=RRRayHit.Distance else RRDist=0 end
		if RRayHit then RDist=RRayHit.Distance else RDist=0 end

		--if RDist==OldRDist and CarModel.DriveSeat.Throttle==1 then reward+=0.5 end
		if LDist==OldLDist and CarModel.DriveSeat.Throttle==1 and CarModel.DriveSeat.AssemblyLinearVelocity.Magnitude > 5 then reward+=5
		elseif LDist==OldLDist and CarModel.DriveSeat.Throttle==1 then reward+=0.5 end

		environmentVector = {
			
			{
				1,
				
				FLDist, FDist, FRDist, LDist, RiDist,-- RLDist, RDist, RRDist,
				
				currRotationErrorY, goalReached,
				
				math.clamp(CarModel.DriveSeat.ThrottleFloat, -1, 1), math.clamp(CarModel.DriveSeat.SteerFloat, -1, 1)
			
			}
			
		}
		
		print('Review: ', environmentVector, reward)
		reward = 0 --getRewardValue(environmentVector, predictedLabel)
		
		if touched then
			reward=defaultPunishment
			print('Punishment: ', reward)
			--Model:getActorModel():reset()
			--Model:getCriticModel():reset()
			reward+=inverseGoalDistance
		elseif (currentPosition-previousPosition).Magnitude > 0 and CarModel.DriveSeat.Throttle==1  then
			reward+=inverseGoalDistance
		end
		
		if goalReached==1 and CarModel.DriveSeat.Throttle<=0 then
			reward+=5
		end
		
		totalReward+=reward
		predictedLabel, confidence = Model:reinforce(environmentVector, reward)
		print('PredictedLabel = ', predictedLabel)
		--print('PREDICTED RESULT: ', predictedLabel, 'REWARDED: ', isRewarded)
		
		print('Reward: ', reward, confidence)
		Controller[predictedLabel]()
			
		
		--table.insert(isRewardedArray, isRewarded)

		--currentAverageAccuracy = getCurrentAverageAccuracy()

		--if (currentAverageAccuracy ~= nil) then print(currentAverageAccuracy) end
		
		if touched then
			CarModel:Destroy()
			game:GetService('RunService'):UnbindFromRenderStep("DrawGizmos")
			break
		end

	end

	startEnvironment(Model, buildCar())
	
end

local function run()

	local QDNModel = buildModel()
	local CarModel = buildCar()

	startEnvironment(QDNModel, CarModel)

end

run()

Here is the code

#652

Oh wait. This is the problem with your code.


	local FLDist
	local FRDist
	local LDist
	local RiDist
	local FDist
	local RLDist
	local RRDist
	local RDist

Should have at least set some values if it cant get a ray hit.

Also update the library, I just fixed the experience replays.

1 Like
#653

image
Still the same, in the while loop I factored this in already as well

#654

I’m out of ideas. You’ll have to give me the full source code in a private message if you want this fixed.

I’m seriously not picking up any errors for those algorithms in my “Library Lab” place.

#655

That is the full source code :sweat_smile:

#656

I mean the whole roblox file. I can’t do proper analysis without the things that are interacting in workspace.

#657

That being said, try adding pcall around :reinforce() first. I want to see if it continues working or not.

#658

It works fine with a pcall
image

#659

Okay yeah. Definitely the calculation is too slow for the neural network.

You need to add some sort of debounce to prevent this from happening.

#660

could I increase the loop’s task.wait() time? or is there a better alternative

#661

It’s better if you do this


while true do

-- Your code

task.wait()

end

This way you won’t have any idle times and also allows the code to be fully processed.

1 Like
#662

Still errors out even when I task.wait(1) this loop structure

#663

Hmm… I see then. So you’ll have to wrap it around in pcall then. No other way around I suppose. Similar what I did to the sword fighting AIs code.

1 Like
#664

Hmm, the thing is, it doesn’t reinforce a single time, so it just plays the first control over and over again, I don’t think I can use this specific model then, I’ll need to find one that can compute it, which was A2C I think. I’d suppose this is a limitation of roblox Luau, I heard roblox has a way to make computing run smoother, through the usage of Actors, I have not tried it myself but you could experiment with it if you want.

Yep, as expected A2C does work without pcalls, hmm weird.

#665

Well, I kind of tried that and it didn’t give good results. It technically made the calculations far more slower due to the overhead when communicating between actors/.threads.

1 Like
#666

I think I managed to find some good settings nonetheless, the agent seems to not turn into walls anymore with A2C and surprisingly reaches the goal part on the straight stretch

image
image1184×623 32.2 KB

It might look like its only using its first output, but after further testing I found out that that is in fact not true, I don’t know why its so perfect that it doesn’t want to steer away from the goal but, it certainly steers either to the left or the right which is a case of bad reward function on my part.

It is less random, to say the least, might have to do with my epsilon setting.

#667

Okay, I accidentally skipped over some of what you said and fixed the PPO and PPO clip. hopefully it doesn’t give issues now.

1 Like
#668

PPOClip still gives me this :sob:

image

#669

Oh sorry lol. I forgot to push it to GitHub. Now it should be there.

1 Like
#670

Great improvement I saw, the Agent actually speeds up then tries to slow down by putting its throttle into negative, slowing the car down but not enough so it eventually hits the wall, I will hook up a brake function to let it slow down and see what happens! (PS: not talking abt the video, just trained a fresh agent not too long ago) And I think I figured out why my agent kept turning into the wall a few days ago… you see, my epilson value for reinforcementquicksetup was set above 0 for some reason that made the correct data get manipulated into incorrect data, or so I presume. After setting it to 0, it is much more accurate!

1 Like