Help making custom shadow system

Hey!

  1. What do you want to achieve? I want to make a custom shadow system, specifically a recreation of the old Stencil shadows which were removed in 2014.

  2. What is the issue? I cannot figure out how to get the shadow to be filled in, and I have no clue how I’m gonna start with Meshes, Cylinders and Spheres.

  3. What solutions have you tried so far? I tried winging it, using a basic diagram as my outline, but I soon got stuck on stuff I shouldn’t have.

This is the module I have for this:

local shadow = {}

function shadow.GetVertices(part : BasePart)
	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice1"
	at.Position = Vector3.new(part.Size.X/2,part.Size.Y/2,part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice2"
	at.Position = Vector3.new(-part.Size.X/2,part.Size.Y/2,part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice3"
	at.Position = Vector3.new(-part.Size.X/2,-part.Size.Y/2,-part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice4"
	at.Position = Vector3.new(-part.Size.X/2,-part.Size.Y/2,part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice5"
	at.Position = Vector3.new(-part.Size.X/2,part.Size.Y/2,-part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice6"
	at.Position = Vector3.new(part.Size.X/2,part.Size.Y/2,-part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice7"
	at.Position = Vector3.new(part.Size.X/2,-part.Size.Y/2,part.Size.Z/2)

	local at = Instance.new("Attachment")
	at.Parent = part
	at.Name = "Vertice8"
	at.Position = Vector3.new(part.Size.X/2,-part.Size.Y/2,-part.Size.Z/2)
end

--[[

 t1 =---------= t2
 ||          -=
  |        --
  |      --
  |    --
  |  --
 ||=-
 t3

]]

function shadow.TriangleSurfaceGuiThingy(t1,t2,t3,canvas,face)
	local sg = Instance.new("SurfaceGui")
	sg.Parent = workspace.Shadows
	sg.Name = "RWBX_Shadow"
	sg.Face = face
	sg.SizingMode = Enum.SurfaceGuiSizingMode.PixelsPerStud
	sg.PixelsPerStud = 50
	sg.Adornee = canvas
	
	local img = Instance.new("ImageLabel")
	img.Parent = sg
	img.Name = "RWBX_Tri"
	img.Image = "rbxassetid://73331143039375"
	img.ImageTransparency = 0.8
	img.BackgroundTransparency = 1
	img.AnchorPoint = Vector2.new(0,1)
	img.Size = UDim2.new(0,(t1 - t2).Magnitude*50,0,(t1 - t3).Magnitude*50)
	img.Rotation = 270
	img.Position = UDim2.new(0.5,-t1.Z * 50+canvas.Position.Z - 150,0.5,t1.X * 50+canvas.Position.X - 50)
end

function shadow.GetShadowCorners(corner : Attachment)
	local direction = (game:GetService("Lighting"):GetSunDirection() * Vector3.new(10,10,10))
	local rayStartingPoint = corner.WorldPosition + direction
	
	local rayLookAt = CFrame.new(rayStartingPoint,corner.WorldPosition)
	local rayParams = RaycastParams.new()
	rayParams.FilterType = Enum.RaycastFilterType.Exclude
	rayParams.FilterDescendantsInstances = {corner.Parent}
	
	local ray = workspace:Raycast(rayStartingPoint,rayLookAt.LookVector * 5000000,rayParams)
	
	local part = Instance.new("Part",workspace.Parts)
	part.Size = Vector3.new(.1,.1,.1)
	part.Position = ray.Position
	part.Anchored = true
end

function shadow.Refresh(part)
	for i,e in pairs(workspace.Parts:GetChildren()) do
		e:Destroy()
	end
	
	for i,e in pairs(part:GetChildren()) do
		e:Destroy()
	end
	
	shadow.GetVertices(part)
	
	for i,e in pairs(part:GetChildren()) do
		shadow.GetShadowCorners(e)
	end
end

return shadow

I would deeply appreciate any help I can get on this, because I’ve been wanting to get these style of shadows for at least a few months now.

Thanks in advance! (and of course, if you need any more information I will gladly send it)

Well basically you can’t. Those old stencil shadows used a completely separate render pass to check if a pixel was inside a shadows volume or not and you cant really do that in Roblox, so yeah it isn’t really possible. :slightly_frowning_face:

I know how I will do it, I just don’t know how I’m gonna fill in the shadow. I know where the shadow is though, and I’m sure I could figure it out for things like Spheres and Heads.

I know how to make heads and spheres. But I build them myself. cause I can’t script