InventoryUI Drag Offset Issue

I want to have a drag-based inventory system. To do this, I’ve written the following code and have made use of this UIDragModule.

The issue is that the UI elements get offset from the mouse as they get teleported back to their old position before they got put in with the UIGridLayout. An example of this behaviour is found below.

I’d like the drag to always start at the position of where the actual item is, instead of where it was when the user stopped dragging, however, I can’t seem to figure out how to do this.

Help is appreciated!

local InventoryServiceClient = {}

local OldPositionInfo = {}

-- // Services
local Connection = require(game.ReplicatedStorage.Modules.Connection)
local InventoryRemote = Connection:Get("RemoteEvent", "Inventory")

local UIDragService = require(script.Parent.UIDragService)
local UIS = game:GetService("UserInputService")

-- // Variables
local player: Player = game.Players.LocalPlayer
local playerGui: PlayerGui = player:WaitForChild("PlayerGui")
local CompleteInventoryGui: Frame = playerGui:WaitForChild("InventoryGui").Inventory

local inventoryGui: ScrollingFrame = CompleteInventoryGui.MainInventoryScrollFrame
local containerGui: ScrollingFrame = CompleteInventoryGui.ContainerInventoryScrollFrame

-- // Local Functions

local function UpdateInventory()
	local Backpack = player:FindFirstChild("Backpack")
	
	-- Clear previous GUI items
	for _,GuiObject in pairs(inventoryGui:GetChildren()) do
		if GuiObject:IsA("ImageButton") then
			GuiObject:Destroy()
		end
	end
	
	-- For each item in the backpack
	if not Backpack then return end
	for _,Item in pairs(Backpack:GetChildren()) do
		if Item:IsA("Tool") then
			local ItemConfig = Item:FindFirstChild("ItemConfiguration") or script.PlaceholderConfig
			
			local ItemName = Item.Name
			
			local GuiItemObject = script.ItemObject:Clone()
			GuiItemObject.Parent = inventoryGui
			GuiItemObject.ItemDescription.Text = ItemConfig.Description.Value
			GuiItemObject.ItemTitle.Text = ItemName
			GuiItemObject.ItemImage.Image = ItemConfig.Image.Texture
			
			local DraggableItemObject = UIDragService.new(GuiItemObject)
			DraggableItemObject:Enable()
			
			
			local OldGuiObjectPosition = nil;
			local OldParent = nil;
		
			-- // User started dragging the item.
			DraggableItemObject.DragStarted = function()
				OldParent = GuiItemObject.Parent
				GuiItemObject.Parent = inventoryGui.Parent
				--if OldPositionInfo[GuiItemObject] then
				--	GuiItemObject.Position = OldPositionInfo[GuiItemObject]
				--end
			end
			
			-- // User is dragging the item
			DraggableItemObject.Dragged = function(position)
			end

			-- // User stopped dragging the item
			DraggableItemObject.DragEnded = function()
				local mousePos = UIS:GetMouseLocation()
				
				-- If mouse is in main inventory
				if mousePos.X >= inventoryGui.AbsolutePosition.X and 
					mousePos.X <= inventoryGui.AbsolutePosition.X + inventoryGui.AbsoluteSize.X and 
					mousePos.Y >= inventoryGui.AbsolutePosition.Y and 
					mousePos.Y <= inventoryGui.AbsolutePosition.Y + inventoryGui.AbsoluteSize.Y then
					GuiItemObject.Parent = inventoryGui
					GuiItemObject.LayoutOrder = #inventoryGui:GetChildren()
					
					for _,v in inventoryGui:GetChildren() do
						if not v:IsA("UIGridLayout") then
							v.LayoutOrder = v.LayoutOrder - 1
						end
					end
						
				-- If mouse is in other inventory
				elseif mousePos.X >= containerGui.AbsolutePosition.X and
					mousePos.X <= containerGui.AbsolutePosition.X + containerGui.AbsoluteSize.X and
					mousePos.Y >= containerGui.AbsolutePosition.Y and
					mousePos.Y <= containerGui.AbsolutePosition.Y + containerGui.AbsoluteSize.Y then
					GuiItemObject.Parent = containerGui
					GuiItemObject.LayoutOrder = #containerGui:GetChildren()
					
					for _,v in containerGui:GetChildren() do
						if not v:IsA("UIGridLayout") then
							v.LayoutOrder = v.LayoutOrder - 1
						end
					end
				else
					GuiItemObject.Parent = OldParent
				end
				
				--OldPositionInfo[GuiItemObject] = GuiItemObject.Position
				
				local totalItems = #GuiItemObject.Parent:GetChildren()
			end
		end
	end
end

local function Init()
	local playerBackpack = player:WaitForChild("Backpack")
	
	playerBackpack.ChildAdded:Connect(function(tap)
		UpdateInventory()
	end)

	playerBackpack.ChildRemoved:Connect(function()
		UpdateInventory()
	end)
	
	UpdateInventory()
end





-- // Global Functions

InventoryRemote:ConnectCallback(function(func, ...)
	if func == "Init" then
		Init()
	end
end)

-- // Extra Code
return InventoryServiceClient