MongoDB Integration & Environment Modules

Help and Feedback Code Review
, ,
1

I’ve written two modules - MongoClient and Environment.

MongoClient aims to provide a lua-friendly interface to MongoDB, making use of their Data API and HttpService. As part of this process you need to keep track of configuration information (some of which is sensitive). These being your API key and app id.

This is where the Environment module comes in. It uses datastores and memorystores to save this configuration information so that you can keep it out of your source code. Note that this does not aim to prevent hackers / exploiters from accessing this information but rather to make the sharing of source code more secure / encourage good practices when handling API keys.

Potential Improvements

I have thought about some kind of caching and aggregation for the data api requests but I’m not sure if that should be included in this module or a higher level module. This would group similar module requests together into fewer http requests, aiming to be more efficient with your specific Mongo DB access limits.

I’m also not sure if it is necessary to use both a datastore and a memorystore. My idea was that relying solely on a datastore may lead to multiple servers accessing the same keys at the same time which would cause throttling.

MongoDB Dev.rbxl (39.3 KB)

Example usage:

local CLUSTER_NAME = "Cluster0"

local sStore = game:GetService("ServerStorage")

local MongoClient = require(sStore:WaitForChild("MongoClient"))
local Environment = require(sStore:WaitForChild("Environment"))

-- Find API key and App id from the Data API (go to https://cloud.mongodb.com/v2/ )
local API_KEY = Environment["MONGO_DB_API_KEY"] -- Replace with your api key
local APP_ID = Environment["MONGO_DB_APP_ID"] -- Replace with your app id

local client = MongoClient.new(API_KEY, APP_ID)

local documents = client:FindOne(CLUSTER_NAME, "test-db", "test-collection",
	{
		name = "Alice"
	}
)

print(documents)
MongoClient Source 
local BASE_URL = "https://data.mongodb-api.com/app/%s/endpoint/data/v1"
local ACTION_PATH = "/action/"

local FIND_ONE_ACTION = "findOne"
local FIND_ACTION = "find"

local INSERT_ONE_ACTION = "insertOne"
local INSERT_ACTION = "insertMany"

local UPDATE_ONE_ACTION = "updateOne"
local UPDATE_ACTION = "updateMany"

local REPLACE_ONE_ACTION = "replaceOne"

local DELETE_ONE_ACTION = "deleteOne"
local DELETE_ACTION = "deleteMany"

local AGGREGATION_ACTION = "aggregate"

local HTTPService = game:GetService("HttpService")

local function getHeaders(apiKey)
	return {
		["Content-Type"] = "application/json",
		["api-key"] = apiKey,
		["Access-Control-Request-Headers"] = "*"
	}
end

local function postRequest(url, headers, data)
	local success, response = pcall(function()
		return HTTPService:RequestAsync{
			Method = "POST",
			Url = url,
			Headers = headers,
			Body = HTTPService:JSONEncode(data)
		}
	end)
	
	if not success then
		warn(response)
		return nil
	end

	return response	
end

local MongoClient = {}
MongoClient.__index = MongoClient

function MongoClient.new(apiKey, appId)
	local client = {}
	client.Headers = getHeaders(apiKey)
	client.Url = string.format(BASE_URL, appId)
	
	return setmetatable(client, MongoClient)
end

function MongoClient:FindOne(dataSource, database, collection, filter, projection)
	--[[
	{
	  "dataSource": "<data source name>",
	  "database": "<database name>",
	  "collection": "<collection name>",
	  "filter": <query filter>,
	  "projection": <projection>
	}
	]]
	
	local response = postRequest(self.Url .. ACTION_PATH .. FIND_ONE_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			filter = filter,
			projection = projection
		})
	
	if response == nil or response.StatusCode ~= 200 then
		return nil
	end
	
	return HTTPService:JSONDecode(response["Body"])["document"]
end

function MongoClient:FindMany(dataSource, database, collection, filter, projection, sort, limit, skip)
	--[[
	{
	  "dataSource": "<data source name>",
	  "database": "<database name>",
	  "collection": "<collection name>",
	  "filter": <query filter>,
	  "projection": <projection>,
	  "sort": <sort expression>,
	  "limit": <number>,
	  "skip": <number>
	}
	]]
	local response = postRequest(self.Url .. ACTION_PATH .. FIND_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			filter = filter,
			projection = projection,
			sort,
			limit,
			skip
		}
	)
	
	if response == nil or response.StatusCode ~= 200 then
		return {}
	end

	return HTTPService:JSONDecode(response["Body"])["documents"]
end

function MongoClient:InsertOne(dataSource, database, collection, document)
	local response = postRequest(self.Url .. ACTION_PATH .. INSERT_ONE_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			document = document
		})

	if response == nil or response.StatusCode ~= 200 then
		return nil
	end

	return response
end

function MongoClient:InsertMany(dataSource, database, collection, documents)
	local response = postRequest(self.Url .. ACTION_PATH .. INSERT_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			documents = documents
		})

	if response == nil or response.StatusCode ~= 200 then
		return nil
	end

	return response
end

function MongoClient:UpdateOne(dataSource, database, collection, filter, update, upsert)
	--[[
	{
	  "dataSource": "<data source name>",
	  "database": "<database name>",
	  "collection": "<collection name>",
	  "filter": { ... },
	  "update": { ... },
	  "upsert": true|false
	}
	]]
	
	local response = postRequest(self.Url .. ACTION_PATH .. UPDATE_ONE_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			filter = filter,
			update = update,
			upsert = upsert
		}
	)
	
	return response
end

function MongoClient:UpdateMany(dataSource, database, collection, filter, update, upsert)
	--[[
	{
	  "dataSource": "<data source name>",
	  "database": "<database name>",
	  "collection": "<collection name>",
	  "filter": { ... },
	  "update": { ... },
	  "upsert": true|false
	}
	]]

	local response = postRequest(self.Url .. ACTION_PATH .. UPDATE_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			filter = filter,
			update = update,
			upsert = upsert
		}
	)

	return response
end

function MongoClient:DeleteOne(dataSource, database, collection, filter)
	--[[
	{
	  "dataSource": "<data source name>",
	  "database": "<database name>",
	  "collection": "<collection name>",
	  "filter": <query filter>
	}
	]]

	local response = postRequest(self.Url .. ACTION_PATH .. DELETE_ONE_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			filter = filter
		})

	if response == nil or response.StatusCode ~= 200 then
		return nil
	end

	return response
end

function MongoClient:DeleteMany(dataSource, database, collection, filter)
	--[[
	{
	  "dataSource": "<data source name>",
	  "database": "<database name>",
	  "collection": "<collection name>",
	  "filter": <query filter>
	}
	]]

	local response = postRequest(self.Url .. ACTION_PATH .. DELETE_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			filter = filter
		})

	if response == nil or response.StatusCode ~= 200 then
		return nil
	end

	return response
end

function MongoClient:Aggregate(dataSource, database, collection, pipeline)
	local response = postRequest(self.Url .. ACTION_PATH .. AGGREGATION_ACTION,
		self.Headers,
		{
			dataSource = dataSource,
			database = database,
			collection = collection,
			pipeline = pipeline
		})

	if response == nil or response.StatusCode ~= 200 then
		return nil
	end

	return response
end

return MongoClient
Environment Source 
local DATASTORE_NAME = "ENV_DS"
local MEMSTORE_NAME = "ENV_MS"

local DS_KEY = "ENV"
local MS_KEY = "ENV_%03i"

local MAX_MS_KEYS = 200
local MS_EXPIRATION = 60

local MAX_RETRIES = 10
local RETRY_INTERVAL = 1

local MS_KEY_CHAR_LIMIT = 128 -- 128 chars
local MS_VALUE_SIZE_LIMIT = 30_000 -- 32KB strict limit

local DataStoreService = game:GetService("DataStoreService")
local MemoryStoreService = game:GetService("MemoryStoreService")
local HttpService = game:GetService("HttpService")

local envDataStore = DataStoreService:GetDataStore(DATASTORE_NAME)
local envMemStore = MemoryStoreService:GetSortedMap(MEMSTORE_NAME)

local function getEnvFromDS()
	return pcall(function()
		return envDataStore:UpdateAsync(DS_KEY, function(currentValue)
			if currentValue == nil then
				return {}
			else
				return currentValue
			end
		end)
	end)
end

local function updateDS(environment)
	return pcall(function()
		return envDataStore:UpdateAsync(DS_KEY, function(currentValue)
			return environment
		end)
	end)
end

local function getEnvFromMS()
	return pcall(function()
		return envMemStore:GetRangeAsync(
			Enum.SortDirection.Ascending,
			200
		)
	end)
end

local function sendChunk(key, chunk)
	return pcall(function()
		return envMemStore:UpdateAsync(
			key,
			function(currentValue)
				return HttpService:JSONEncode(chunk)
			end,
			MS_EXPIRATION)
	end)
end

local function updateMS(environment)
	local index = 0
	local success = true
	local chunk = {}

	for key, value in pairs(environment) do
		chunk[key] = value
		
		-- Check size of chunk
		local encoded = HttpService:JSONEncode(chunk)
		if #encoded < MS_VALUE_SIZE_LIMIT then continue end
		
		-- Send the next chunk to MS
		chunk[key] = nil
		encoded = HttpService:JSONEncode(chunk)
		assert(#encoded > 2 and #encoded < MS_VALUE_SIZE_LIMIT)
		
		index += 1
		local key = string.format(MS_KEY, index)
		local _success = sendChunk(key, chunk)
		
		chunk[key] = value
	end
	

	local encoded = HttpService:JSONEncode(chunk)
	if #encoded > 2 then
		index += 1
		local key = string.format(MS_KEY, index)
		local _success = sendChunk(key, chunk)
	end
	return success
end

local function updateEnv(environment)
	local msSuccess = updateMS(environment)
	local dsSuccess = updateDS(environment)
	return msSuccess and dsSuccess
end

local function init()
	
	local memSuccess, value = getEnvFromMS()
	if memSuccess and #value == 0 then
		local dsSuccess, dsValue = getEnvFromDS()
		local retries = 0
		while not dsSuccess and retries < MAX_RETRIES do
			dsSuccess, dsValue = getEnvFromDS()
			retries += 1
			task.wait(RETRY_INTERVAL)
		end

		updateMS(dsValue)
		return dsValue
	elseif memSuccess then
		local env = {}
		for _, envChunk in ipairs(value) do
			local decodedChunk = HttpService:JSONDecode(envChunk.value)
			for key, value in pairs(decodedChunk) do
				env[key] = value
			end
		end
		return env
	end
end

local Environment = init()

local EnvMeta = {}
EnvMeta.__index = EnvMeta

function EnvMeta.__index(_, key)
	return Environment[key]
end

function EnvMeta.__newindex(_, key, value)
	assert(type(value) == "string", "Environment keys must be strings")
	assert(#key <= MS_KEY_CHAR_LIMIT, "key length too long")
	assert(#HttpService:JSONEncode(value) <= MS_VALUE_SIZE_LIMIT, "value too large")

	Environment[key] = value
	updateEnv(Environment)
end

return setmetatable({}, EnvMeta)
2 Likes
2

Wonderful and very clever implementation of MongoDB in Roblox. I could see this being very useful for all sorts of projects. However, I do worry about security. I’ve never been a huge user of MongoDB so I may be incorrect, but I feel like trying to link MongoDB with Roblox could pose some security issues in terms of data being taken, and manipulated before it reaches the database. Apart from that, amazing work.

1 Like