Chunk system uses a very big amount of ram

Help and Feedback Scripting Support
#1

I’m writing a minecraft-ish game, with a chunk system.

Each chunk has a 16x256x16 size, which should be pretty small and easy to store, however with storing air (I really need to store air blocks) it takes 60.58 MiB of RAM for ONE CHUNK. I need at least 169 chunks per player, which uses ~9.9 GiB of RAM, and roblox servers only have 6 GiB of RAM, 1 of it being used by random roblox features.

Without storing air blocks it uses 30 MiB of RAM per chunk, but without storing air blocks I would have to work way harder to get the same thing done.

One ideea I have is to generate and delete chunks on the fly, and only save in a array the changes made by a player, however I would have to regenerate the chunk every time a block changes, making it extremely slow.

Any ideea on how to optimise this? This is the main culprit:


function Terrain:GenerateChunk(chunkPosition, onFinished)
	local chunkNeighbors = {}

	for neightbor_x = -1, 1 do
		for neightbor_z = -1, 1 do
			for neightbor_y = -1, 1 do
				if neightbor_x ~= 0 or not (neightbor_x == neightbor_y and neightbor_x == neightbor_z and neightbor_z == neightbor_y) then
					local neighbor_position = Vector3.new(chunkPosition.X + neightbor_x, chunkPosition.Y + neightbor_y, chunkPosition.Z + neightbor_z)
					table.insert(chunkNeighbors, neighbor_position)
				end
			end
		end
	end

	local chunk = {
		position = chunkPosition,
		blocks = {},
		neighbors = chunkNeighbors,
	}

	local start_x = Terrain.chunk_size.X * chunkPosition.X
	local end_x = (Terrain.chunk_size.X) * (chunkPosition.X + 1)
	local start_z = Terrain.chunk_size.Z * chunkPosition.Z
	local end_z = (Terrain.chunk_size.Z ) * (chunkPosition.Z + 1)
	local start_y = (Terrain.chunk_size.Y ) * (chunkPosition.Y)
	local end_y = (Terrain.chunk_size.Y ) * (chunkPosition.Y + 1)

	for x = start_x, end_x do		
		for z = start_z, end_z do			
			if chunkPosition.Y == 0 then
				local max_y = math.floor((math.noise(x/self.scale, z/self.scale, self.seed) * self.amplitude) + 100)
				local max_block_y = chunkPosition.Y * Terrain.chunk_size.Y + max_y

				for y = chunkPosition.Y * Terrain.chunk_size.Y, max_block_y do
					local material = "unknown"

					if y == max_block_y - 1 then
						material = "grass"
					elseif y < max_block_y - 12 then
						material = "stone"
					elseif y >= max_block_y - 12 then
						material = "dirt"
					end

					local blockPosition = Vector3.new(x, y, z)
					local block = {
						position = blockPosition,
						visible = false,
						rendered = false,
						render_part = nil,
						neighbors = {},
						material = material,
					}

					chunk.blocks[blockPosition] = block
				end

				for y = max_block_y, ((chunkPosition.Y + 1) * Terrain.chunk_size.Y) do					
					local blockPosition = Vector3.new(x, y, z)
					local block = {
						position = blockPosition,
						visible = false,
						rendered = false,
						render_part = nil,
						neighbors = {},
						material = "air",
					}


					chunk.blocks[blockPosition] = block
				end
			else

			end
		end
	end

	for x = start_x, end_x do		
		for z = start_z, end_z do			
			for y = (chunkPosition.Y * Terrain.chunk_size.Y), ((chunkPosition.Y + 1) * Terrain.chunk_size.Y) do
				local block = chunk.blocks[Vector3.new(x, y, z)]
				if block then
					if x == start_x or x == end_x or z == start_z or z == end_z then
						--block.visible = true
					end

					local possibleNeighbors = {
						Vector3.new(x-1,y-1,z-1),
						Vector3.new(x-1,y,z-1),
						Vector3.new(x-1,y+1,z-1),
						Vector3.new(x-1,y-1,z),
						Vector3.new(x-1,y,z),
						Vector3.new(x-1,y+1,z),
						Vector3.new(x-1,y-1,z+1),
						Vector3.new(x-1,y,z+1),
						Vector3.new(x-1,y+1,z+1),
						Vector3.new(x,y-1,z-1),
						Vector3.new(x,y,z-1),
						Vector3.new(x,y+1,z-1),
						Vector3.new(x,y-1,z),
						Vector3.new(x,y+1,z),
						Vector3.new(x,y-1,z+1),
						Vector3.new(x,y,z+1),
						Vector3.new(x,y+1,z+1),
						Vector3.new(x+1,y-1,z-1),
						Vector3.new(x+1,y,z-1),
						Vector3.new(x+1,y+1,z-1),
						Vector3.new(x+1,y-1,z),
						Vector3.new(x+1,y,z),
						Vector3.new(x+1,y+1,z),
						Vector3.new(x+1,y-1,z+1),
						Vector3.new(x+1,y,z+1),
						Vector3.new(x+1,y+1,z+1),
					}
					
					for _, neighbor_position in ipairs(possibleNeighbors) do
						table.insert(block.neighbors, neighbor_position)
						
						local real_block = chunk.blocks[neighbor_position]
						if not real_block then
							for _, chunkNeighbor in ipairs(chunkNeighbors) do
								local chunkNeighbor = self.chunks[chunkNeighbor] 
								if chunkNeighbor then
									if chunkNeighbor.blocks[neighbor_position] then
										real_block = chunkNeighbor.blocks[neighbor_position]
										break
									end
								end
							end
						end
						
						if not real_block then
							block.visible = true

							if block.material == "dirt" then
								block.material = "grass"
							end
							
							break
						end
					end
				end
			end
		end

		--[[if x % 50 == 0 then
			task.wait()
		end]]
	end

	--task.synchronize()
	self.chunks[chunkPosition] = chunk
	onFinished()
end
#2

I’ve tried to debug it further, but it still uses the same amount of RAM

#3

RLE? a compressed data format to store the chunk data

#4

I’m pretty sure that I would need to decode and reencode the chunk every time I change anything with it, which would mean over 5 times per second, that would create a lot of lag, would’t it?

#5

Not sure, you could try? Ultimately, the choice of compression technique would depend on the specific requirements of the application and the characteristics of the data being compressed.

#6

Ill rewrite the way I’m storing chunks and blocks, and Ill make a message once I try your suggestion

1 Like
#7

For the code you could do:

  1. Remove the unnecessary conditional in the neighbor generation loop (simplify basically)
  2. Avoid recalculating the same values in the nested for loops
  3. Reuse table inserts instead of creating new tables
1 Like
#8

A few ideas.

In general:
You can use fewer tables.
Or less data in tables.

Maybe if some data is only applicable if a part has been instantiated, put that data in the part and not the table.

I don’t understand all the code but there is a lot about neighbors. If the code stores information about which neighbors exist, it will be much better to check the neighbours at runtime instead of storing extra information.

Maybe table.create() is more efficient than table.insert() - don’t know.

1 Like
#9

with a few hours of refactoring the entire code, I was able to improve the noise function (add overhangs and such), decrease the size of a chunk from 60MiB to 6MiB, and increase the speed from 600ms to less than 150ms.

I will also use a compression algorithm to store unused chunks in memory.

I gotta thank all of you for supporting me, and if anyone needs it, this is the new code:

local Terrain = {}
Terrain.blockSize = 2
Terrain.chunkSize = Vector3.new(16, 255, 16)
Terrain.splinePositions = {
	[0] = -85,
	[0.1] = -50,
	[0.2] = -50,
	[0.3] = -25,
	[0.4] = -25,
	[0.5] = 0,
	[0.6] = 0,
	[0.7] = 35,
	[0.8] = 50,
	[0.9] = 85,
	[1] = 85
}

local TerrainObject = {}
TerrainObject.__index = TerrainObject

function Terrain.new(seed, scale, amplitude)
	local newTerrain = {}
	setmetatable(newTerrain, TerrainObject)
	
	newTerrain.rng = Random.new(seed)
	newTerrain.scale = scale
	newTerrain.amplitude = amplitude
	newTerrain.chunks = {}
	newTerrain.seed = seed
	
	return newTerrain
end

function TerrainObject:__calculateSpline(neighbors)
	if #neighbors == 0 then
		return {
			["spline"] = 0.55, 
			["value"] = Terrain.splinePositions[0.5]
		}
	end
	
	local neighborsValues = 0
	local totalNeighbors = 0
	for _, neighbor in ipairs(neighbors) do
		neighbor = self.chunks[neighbor]
		if neighbor then
			totalNeighbors += 1
			neighborsValues += neighbor.spline
		end
	end
	
	if totalNeighbors == 0  then
		return {
			["spline"] = 0.55, 
			["value"] = Terrain.splinePositions[0.5]
		}
	end
	
	local spline = math.clamp((neighborsValues / totalNeighbors) + self.rng:NextInteger(0, 50) / 1000, 0, 1)
	local value = math.huge
	
	for otherNumber, _ in pairs(Terrain.splinePositions) do
		if math.abs(spline - otherNumber) < math.abs(spline - value) then
			value = otherNumber
		end
	end
	
	return {
		["spline"] = spline, 
		["value"] = Terrain.splinePositions[value]
	}
end

function TerrainObject:GenerateChunk(position, onFinished)
	if not self.chunks[position.X] then self.chunks[position.X] = {} end
	if not self.chunks[position.X][position.Y]  then self.chunks[position.X][position.Y] = {} end
	
	local chunkNeighbors = table.create(26)

	for neightbor_x = -1, 1 do
		for neightbor_z = -1, 1 do
			for neightbor_y = -1, 1 do
				if neightbor_x ~= 0 or not (neightbor_x == neightbor_y and neightbor_x == neightbor_z and neightbor_z == neightbor_y) then
					local neighbor_position = Vector3.new(position.X + neightbor_x, position.Y + neightbor_y, position.Z + neightbor_z)
					table.insert(chunkNeighbors, neighbor_position)
				end
			end
		end
	end
	
	local spline = self:__calculateSpline(chunkNeighbors)
	local chunk = {
		position = position,
		blocks = table.create(Terrain.chunkSize.X),
		spline = spline.spline,
		splineValue = spline.value,
		neighbors = chunkNeighbors,
	}
	
	for x = 0, Terrain.chunkSize.X do
		if not chunk.blocks[x] then chunk.blocks[x] = table.create(Terrain.chunkSize.Z) end	
		local real_x = position.X * Terrain.chunkSize.X + x
		
		for z = 0, Terrain.chunkSize.Z do
			if not chunk.blocks[x][z] then chunk.blocks[x][z] = table.create(Terrain.chunkSize.Y) end
			local real_z = position.Z * Terrain.chunkSize.Z + z
			
			for y = 75, Terrain.chunkSize.Y do
				local real_y = position.Y * Terrain.chunkSize.Y + y
				
				local density_x = math.noise((real_y / self.scale), (real_z / self.scale), self.seed) * self.amplitude
				local density_y = math.noise((real_x / self.scale), (real_z / self.scale), self.seed) * self.amplitude
				local density_z = math.noise((real_x / self.scale), (real_y / self.scale), self.seed) * self.amplitude
				
				local density = y + spline.value + (density_x + density_y + density_z) / 3
				
				local block = {
					position = Vector3.new(real_x, real_y, real_z),
					light = 0,
					visible = false,
					material = "unknown",
				}
				
				if density < 130 then
				--if density < 130 and density > 85 then
					block.material = "stone"
				else
					block.material = "air"
				end
				
				chunk.blocks[x][z][y] = block
			end
			
			for y = 0, 75 do
				local real_y = position.Y * Terrain.chunkSize.Y + y
				
				local block = {
					position = Vector3.new(real_x, real_y, real_z),
					light = 0,
					visible = false,
					material = "stone",
				}

				chunk.blocks[x][z][y] = block
			end
		end
		
		if x % 10 == 0 then
			task.wait()
		end
	end
	
	for x = 0, Terrain.chunkSize.X do
		for z = 0, Terrain.chunkSize.Z do
			for y = 0, Terrain.chunkSize.Y do
				local block = chunk.blocks[x][z][y]
				
				if block.material ~= "air" then
					--[[local topBlock = chunk.blocks[x][z][y + 1]
					if not topBlock or topBlock.material == "air" then
						block.visible = true
					end]]
				else
					for neighbor_x = -1, 1 do
						for neighbor_y = -1, 1 do
							for neighbor_z = -1, 1 do
								if neighbor_z ~= 0 or not (neighbor_z == neighbor_x and neighbor_z == neighbor_y and neighbor_x == neighbor_y) then
									local real_x = x + neighbor_x
									local real_y = y + neighbor_y
									local real_z = z + neighbor_z
									
									if chunk.blocks[real_x] and chunk.blocks[real_x][real_z] then
										local neighbor = chunk.blocks[real_x][real_z][real_y]
										if neighbor then
											neighbor.visible = true
										end
									end
								end
							end
						end
					end
				end
			end
		end
		
		if x % 10 == 0 then
			task.wait()
		end
	end
	
	self.chunks[position.X][position.Y][position.Z] = chunk
	onFinished()
end

function TerrainObject:LoadChunk(position, onFinished)
	local chunk = self.chunks[position.X][position.Y][position.Z]
	
	for x = 0, Terrain.chunkSize.X do
		for z = 0, Terrain.chunkSize.Z do
			for y = 0, Terrain.chunkSize.Y do
				local block = chunk.blocks[x][z][y]
				local real_x = (position.X * Terrain.chunkSize.X + x) * Terrain.blockSize
				local real_y = (position.Y * Terrain.chunkSize.Y + y) * Terrain.blockSize
				local real_z = (position.Z * Terrain.chunkSize.Z + z) * Terrain.blockSize	
				
				if block.visible and block.material ~= "air" then					
					local Part = Instance.new("Part")
					Part.Anchored = true
					Part.Size = Vector3.new(Terrain.blockSize, Terrain.blockSize, Terrain.blockSize)
					Part.CanCollide = true
					Part.Position = Vector3.new(real_x, real_y, real_z)
					Part.Parent = workspace
				end
			end
		end
		
		if x % 25 == 0 then
			task.wait()
		end
	end
	
	onFinished()
end

function TerrainObject:UnloadChunk(position)
	
end

return Terrain
#10

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.