Relative Position With Viewportframe not working properly

I want to make a gui for a vehicle (out of the toolbox) which shows up objects near the car. Its not working correctly, position is, rotation isnt.

Screenshot with Wall 1:

Screenshot with Wall 1:

Screenshot with Wall 2:

I already tried multiple things. You can see them in the code below (variables that are defined but not used (anymore))

local player = game.Players.LocalPlayer
local shouldUpdate = true

function debugPart(pos)
	local part = Instance.new("Part", workspace.Debug)
	part.Anchored = true 
	part.CanCollide = false
	part.Position = pos
end

function updateRays(carClone)
	local car = player.Character.Humanoid.SeatPart.Parent.Parent
	local raycastParam = RaycastParams.new()
	raycastParam.FilterType = Enum.RaycastFilterType.Blacklist
	
	while task.wait(0.1) and shouldUpdate do
		local partsInViewportFrame = {}
		for _, part in script.Parent.ViewportFrame:GetChildren() do
			if part == carClone then continue end
			if part:IsA("Model") or part:IsA("BasePart") or part:IsA("Folder") then
				part:Destroy()
			end
		end
		for _, attachment in car.HitBox:GetChildren() do
			if not attachment:IsA("Attachment") then continue end
			if #string.split(attachment.Name, " ") == 3 then continue end
			local endPosRay = car.HitBox:FindFirstChild(attachment.Name.." Pos")
			raycastParam.FilterDescendantsInstances = {car, workspace.Debug}
			
			local origin = attachment.WorldPosition
			local destination = endPosRay.WorldPosition
			local direction = destination - origin
			
			local ray = workspace:Raycast(attachment.WorldPosition, direction, raycastParam)
			if not ray then continue end
			if table.find(partsInViewportFrame, ray.Instance) then continue end
			table.insert(partsInViewportFrame, ray.Instance)
			local part = ray.Instance:Clone()
			part.Parent = script.Parent.ViewportFrame
			local relativePositionOfPart = car:GetPivot().Position - part.Position
			local relativeRotationOfPart = car:GetPivot().Rotation - part.Rotation
			local rotationX, rotationY, rotationZ = car:GetPivot().Rotation.X - part.Rotation.X, car:GetPivot().Rotation.Y - part.Rotation.Y, car:GetPivot().Rotation.Z - part.Rotation.Z
			part.CFrame = CFrame.new(relativePositionOfPart) * CFrame.Angles(math.rad(rotationX), math.rad(rotationY), math.rad(rotationZ))
		end
	end
	carClone:Destroy()
end

script.Parent.Parent:GetPropertyChangedSignal("Enabled"):Connect(function()
	if script.Parent.Parent.Enabled == false then 
		shouldUpdate = false print("ended thread")
		return 
	end
	local char = player.Character
	local carClone = char.Humanoid.SeatPart.Parent.Parent:Clone()
	carClone.Parent = script.Parent.ViewportFrame
	carClone:PivotTo(CFrame.new(0,0,0)*CFrame.Angles(math.rad(180),math.rad(180),math.rad(180)))
	local cam = script.Parent.ViewportFrame:FindFirstChildWhichIsA("Camera") or Instance.new("Camera", script.Parent.ViewportFrame)
	cam.CFrame = CFrame.new(Vector3.new(-14.129, 15.402, 24.707), carClone:GetPivot().Position)
	shouldUpdate = true
	updateRays(carClone)
end)

The part where the problem is in the for-loop inside the while-loop inside the updateRays() function.

Thanks for reading!