Try this, I tried to use your variables so you can adapt it better to your code.
function CreateVisualRayPart()
local part = Instance.new("Part", workspace)
part.Anchored = true
part.CanCollide = false
part.Color = Color3.fromRGB(255,0,0)
return part
end
local raycastParams = RaycastParams.new()
raycastParams.FilterDescendantsInstances = {player.Character}
raycastParams.FilterType = Enum.RaycastFilterType.Blacklist
local tip = script.Parent.Tip.Position
local direction = (mousePos - tip).Unit
local magnitude = 300
local visualize = true
local raycastResult = workspace:Raycast(tip, direction * magnitude, raycastParams)
if raycastResult then -- if the ray hit something
if visualize then
local rayPart = CreateVisualRayPart()
rayPart.Size = Vector3.new(0, 0, (tip - raycastResult.Position).Magnitude)
rayPart.CFrame = CFrame.new((tip + raycastResult.Position)/2, raycastResult.Position)
-- this uses CFrame.new(startPosition, lookAt) format
-- finding the middle point between two vectors is like using numbers
-- to find the middle between 5 and 10, you add 5 + 10 and divide by how many numbers (2)
end
-- Rest of code
else -- if the ray didn't hit something
if visualize then
local rayPart = CreateVisualRayPart()
local endPos = tip + (direction * magnitude)
rayPart.Size = Vector3.new(0, 0, magnitude)
rayPart.CFrame = CFrame.new((tip + endPos)/2, endPos)
end
-- Rest of code
end