WorldToViewportPoint not consistent dependent on camera height

Hello, I’m trying to make an imagebutton go to the middle of an object in a viewframe and if I put my camera height to 8000 then it works but when lowering the position of the button is moving more and more to the left. Does anyone know why this is happening and how I could fix this issue?

Code:

local RunService = game:GetService("RunService")

local player = game.Players.LocalPlayer
local character = player.Character or player.CharacterAdded:Wait()
local humanoidRootPart = character:WaitForChild("HumanoidRootPart")

local frame = player.PlayerGui.MainGui.Map
local shipArrow = frame.Ship

local camera = Instance.new("Camera", workspace)
camera.CameraType = Enum.CameraType.Scriptable
frame.CurrentCamera = camera
camera.FieldOfView = 200

function LoadMap()
	for i,v in ipairs(frame:GetChildren()) do
		if v.ClassName == "Part" then v:Destroy() end
	end
	for i,v in ipairs(workspace.Zones:GetChildren()) do
		local temp = v.PrimaryPart:Clone()
		temp.Parent = frame
		
		if game.ReplicatedStorage.Functions.Data.GetData:InvokeServer().CurrentZone == v.Name then
			local vector, inViewport = camera:WorldToViewportPoint(temp.Position)

			local viewportPoint = Vector2.new(vector.X, vector.Y)
			local depth = vector.Z
			player.PlayerGui.MainGui.Map.ImageButton.Position = UDim2.new(viewportPoint.X, 0, viewportPoint.Y, 0)
		end
	end
end

RunService.RenderStepped:Connect(function()
	camera.CFrame = CFrame.new(humanoidRootPart.Position + Vector3.new(0, script.Zoom.Value, 0), humanoidRootPart.Position)
	shipArrow.Rotation = -humanoidRootPart.Orientation.Y - 90
end)

player.PlayerGui.MainGui.Map:GetPropertyChangedSignal("Visible"):Connect(function()
	if not player.PlayerGui.MainGui.Map.Visible then
		LoadMap()
	end
end)

Camera height: 8000
image

Camera height: 6000
image

Camera height: 4000
image

Or if anyone know’s another way I can draw gui elements on the objects in the viewportframe I’m trying to achieve something like this.

Sprite-0001