I made some changes to the scripts. Replace the codes with these ones.
For main script:
local function buildActorModel(ID)
local Model = DataPredict.Models.NeuralNetwork.new(1)
Model:setModelParametersInitializationMode("LeCunUniform")
Model:addLayer(5, true, 'LeakyReLU', 0.01)
Model:addLayer(3, true, 'LeakyReLU', 0.01)
Model:addLayer(7, false, 'StableSoftmax', 0.01)
Model:setClassesList({'A','D','W','S','jump','useWeapon', "none"})
local ModelParameters = loadModelParameters(ActorModelDataStore, ID)
Model:setModelParameters(ModelParameters)
table.insert(ActorModelArray, Model)
return Model
end
local function buildCriticModel(ID)
local Model = DataPredict.Models.NeuralNetwork.new(1)
Model:setModelParametersInitializationMode("LeCunUniform")
Model:addLayer(5, true, 'LeakyReLU', 0.01)
Model:addLayer(3, true, 'LeakyReLU', 0.01)
Model:addLayer(1, false, 'Sigmoid', 0.01)
Model:setClassesList({1, 2})
local ModelParameters = loadModelParameters(CriticModelDataStore, ID)
Model:setModelParameters(ModelParameters)
table.insert(CriticModelArray, Model)
return Model
end
local function buildModel(ID)
local classesList = {'A','D','W','S','jump','useWeapon', "none"}
local MainModel = DataPredict.Models.AdvantageActorCritic.new(60, 0.05, 1)
local AModel = buildActorModel(ID)
local CModel = buildCriticModel(ID)
MainModel:setActorModel(AModel)
MainModel:setCriticModel(CModel)
local MainModelQuickSetup = DataPredict.Others.ReinforcementLearningQuickSetup.new(120, 0.05, 1)
MainModelQuickSetup:setModel(MainModel)
MainModelQuickSetup:setPrintReinforcementOutput(false)
MainModelQuickSetup:setClassesList(classesList)
table.insert(ModelArray, MainModelQuickSetup)
return MainModelQuickSetup
end
For Senses script:
local function getRewardValue(orientationDifference)
local currentHealth = Humanoid.Health
local currentLocation = Character:GetPivot().Position
local currentRotationY = Character:GetPivot().Rotation.Y
local healthChange = currentHealth - previousHealth
local closestEnemy, damageDealt, distanceDifference, distanceToEnemy = getEnemyStatus()
local isSeeingEnemy, viewingDistance = getCurrentView()
local noEnemy = (closestEnemy == nil)
local idlePunishment = (noEnemy and -0.1) or 0
local isEnemyDead = (previousEnemyHealth == 0)
local enemyDeathReward = (isEnemyDead and 1) or 0
local isEnemyReward = (isSeeingEnemy and 10) or 0
local isLookingAtTheWall = ((viewingDistance <= 3) and not isSeeingEnemy)
local isLookingAtTheWallValue = (isLookingAtTheWall and 1) or 0
local isLookingAtTheWallReward = (isLookingAtTheWall and -40) or 0
local heightChange = (currentLocation.Y - previousLocation.Y)
local heightChangeValue = ((math.abs(heightChange/lastDeltaTime) > 1) and 0) or 1
local isInRange = false
local isTooNear = false
if not distanceToEnemy then
isTooNear = (distanceToEnemy <= 2)
isInRange = (distanceToEnemy <= 10)
end
local isWeaponUsed = (lastAction == "useWeapon")
local isDamageDealt = (damageDealt > 0)
local isPlayerHittingEnemyReward = (isWeaponUsed and isInRange and isSeeingEnemy and 30) or (isWeaponUsed and isInRange and 3) or (isWeaponUsed and not isInRange and -0.1) or 0
local movedDistance = (currentLocation - previousLocation).Magnitude
local isWalkingForwardValue = ((lastAction == "W") and 10) or 0
local hasPlayerMovedValue = ((movedDistance/lastDeltaTime > 2) and 1) or 0
local damageDealtRatio = (damageDealt / maxHealth)
local healthPercentage = currentHealth / maxHealth
local healReward = (1 - healthPercentage) * math.log(distanceToEnemy)
local isRotated = (lastAction == "rotateRight") or (lastAction == "rotateLeft")
local rotatingForNoReasonValue = (not isLookingAtTheWallReward and isRotated and -30) or 0
local distanceChangeReward = (isTooNear and -3) or (heightChangeValue * hasPlayerMovedValue * distanceDifference * 10 * isWalkingForwardValue * isEnemyReward) --math.sign(distanceDifference) * math.log(math.abs(distanceDifference))
local rewardValue
if isLookingAtTheWall then
rewardValue = -10
elseif ((lastAction == "S") or (lastAction == "A") or (lastAction == "D") or (lastAction == "AW") or (lastAction == "DW") or (lastAction == "AS") or (lastAction == "DS")) and (not isTooNear) then
rewardValue = -40
else
rewardValue = healReward + damageDealtRatio + enemyDeathReward + distanceChangeReward + isPlayerHittingEnemyReward + isLookingAtTheWallReward + orientationDifference + isEnemyReward --+ changeInOrientationReward
end
return rewardValue
end