Thanks for clarifying! That is good and very interesting to hear. I always get excited when we have a new and more powerful small model such as this! I was testing this alongside Zephyr 7b which has a much shorter context window. Since Zephyr has the same format as this model they can technically be run concurrently to expand the context window of a a Zephyr generated response by further expanding the context window to 2048 tokens such as tinyllama is capable of, Zephyr I think is maxed out at 1024 or 512 tokens.
When I tested it with my chatbot that it really did quite well in completing the Zephyr response!
Example of TinyLlamas coding abilities
<|system|>
You are a ROBLOX studio Chatbot Specializing in Luau code.
<|user|>
Write me a function to calculate the first 10 digits of the fibonacci sequence in Lua and print it out to the CL.
<|assistant|>
Here’s a Lua function that calculates the first 10 digits of the Fibonacci sequence and prints them to the console:
function fibonacci(n)
if n <= 1 then
return n
else
return fibonacci(n-1) + fibonacci(n-2)
end
end
-- Example usage
print(fibonacci(10)) -- Output: 55
API Documentation to use TinyLlama on inference.
local HttpService = game:GetService("HttpService")
function cm.TinyLlama(system,query)
-- Define the API URL and the authorization header
local API_URL = "https://api-inference.huggingface.co/models/TinyLlama/TinyLlama-1.1B-Chat-v1.0"
local headers = {Authorization = bearer}
-- Define a function to query the model
local function queryModel(input, temperature)
-- Create a payload table with the input and the temperature
local payload = {inputs = input, temperature = temperature}
-- Encode the payload table into a JSON string
local payloadJSON = HttpService:JSONEncode(payload)
-- Send a POST request to the API URL with the header and the payload
-- Use pcall to catch any errors
local success, response = pcall(HttpService.PostAsync, HttpService, API_URL, payloadJSON, Enum.HttpContentType.ApplicationJson, false, headers)
-- Check if the request was successful
if success then
-- Decode the response into a table
-- Use pcall to catch any errors
local success, responseTable = pcall(HttpService.JSONDecode, HttpService, response)
-- Check if the decoding was successful
if success then
-- Return the response table
return responseTable
else
-- Return nil and the error message
return nil, responseTable
end
else
-- Return nil and the error message
return nil, response
end
end
-- Define a function to format the response
local function formatResponse(str)
-- Find the assistant response in the string
local start = string.find(str, "<|assistant|>")
local finish = string.len(str)
-- Extract the response
local response = string.sub(str, start + 13, finish)
-- Return the response
return response
end
-- Test the function with an example input
local input = "<|system|>\n"..system.."</s>\n<|user|>\n"..query.."</s>\n<|assistant|>"
local temperature = 2
local output = queryModel(input, temperature)
if output then
output = formatResponse(output[1].generated_text)
print(output)
return output
else
return nil
--print("Something went wrong.")
end
end
Also wanted to share this example of engineering the system message to cultivate self-awareness in addition to expanding the context window using this model in the text generation code.
function cm.ZephyrStory(person,personal,Playername,quer)
-- Define the API URL and the authorization header
local API_URL = "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta"
local headers = {Authorization = bearer}
-- Define the HttpService
local HttpService = game:GetService("HttpService")
-- Define a function that takes an input and queries the model
local function queryModel(input, temperature)
-- Create a payload table with the input and the temperature
local payload={inputs = input, temperature = temperature,max_new_tokens=1000, min_tokens=250, top_k=100, top_p=0.11}
-- local payload = {inputs = input, temperature = temperature}
-- Encode the payload table into a JSON string
local payloadJSON = HttpService:JSONEncode(payload)
-- Send a POST request to the API URL with the header and the payload
-- Use pcall to catch any errors
local success, response = pcall(HttpService.PostAsync, HttpService, API_URL, payloadJSON, Enum.HttpContentType.ApplicationJson, false, headers)
-- Check if the request was successful
if success then
-- Decode the response into a table
-- Use pcall to catch any errors
local success, responseTable = pcall(HttpService.JSONDecode, HttpService, response)
-- Check if the decoding was successful
if success then
-- Return the response table
return response--Table-- return json
else
-- print()
-- Return nil and the error message
return nil, response--Table
end
else
-- Return nil and the error message
return nil, response
end
end
local personality=personal[1]
local awarobserve=personal[2]
--identify..timeod..awareobserve
--local Resulttable,speakers=cm.LocalZephyrStory(str,npcnam,{[1]=persona,[2]=awareobserve,[3]=identity,[4]=timeod},Player)
local identity=personal[3]
local timeod=personal[4]
local insight=personal[5]
local memory=personal[7]
local previousconversation=personal[6]
if previousconversation==nil then
previousconversation=""
else
local function RebuildConversation(tbl,response)--reduce size of system message
--local tbl,speakers,str=cm.LocalZephyrDecode(response)
local sum="" for i,v in tbl do for t,o in v do
if t~="narrator" then
sum=sum.." \n\n "..t..": "..o
else
sum=sum.." \n\n "..o
end end print(sum)
end
sum=sum.." \n\n "..Playername..": "..response
return sum
end
previousconversation=RebuildConversation(personal[6][1])
end
--awareobserve,timeod,identity
-- Test the function with an example input
--cachedconversation
local input = "<|system|>\n "..identity..timeod..insight..awarobserve..memory..". Parse dialogues with "..person..": and "..Playername..": .</s>\n<|"..Playername.."|>\n "..quer.." </s>\n<|assistant|>"..previousconversation
local temperature = 2
local output,Error = queryModel(input,temperature)
print(output)
local iterations=0
local function RebuildResponse(response)--reduce size of system message
local tbl,speakers,str=cm.LocalZephyrDecode(response)
local sum="" for i,v in tbl do for t,o in v do
if t~="narrator" then
sum=sum.." \n\n "..t..": "..o
else
sum=sum.." \n\n "..o
end end print(sum)
end
local input = "<|system|>\n"..identity..memory..awarobserve.." Parse dialogues with "..person..": and "..Playername..": . </s>\n<|"..Playername.."|>\n "..quer.."</s><|assistant|>"..sum
if iterations==2 then
input = "<|system|>\n"..memory..awarobserve..identity.." Parse dialogues with "..person..": and "..Playername..": .</s>\n<|"..Playername.."|>\n "..quer.."</s><|assistant|>"..sum
elseif iterations==3 then
input = "<|system|>\n"..identity.."\n Parse dialogues with "..person..": and "..Playername..": .</s>\n<|"..Playername.."|>\n "..quer.."</s>\n<|assistant|>"..sum
end
return input
end
if not Error then
local function iterateoutput(output)
local checkedinput
local loadoutput
local previnput
repeat
iterations+=1
previnput=HttpService:JSONDecode(output)[1].generated_text
local loadoutput = queryModel(RebuildResponse(output))
if loadoutput~=nil then
checkedinput=HttpService:JSONDecode(loadoutput)[1].generated_text
if checkedinput then--only update output if valid
output=loadoutput
print(output)
else
break
end
else
break
end
until checkedinput==previnput or iterations>=3
return output
end
output=iterateoutput(output)
end
local function DecodeResponse(response)--reduce size of system message
local tbl,speakers,str=cm.LocalZephyrDecode(response)
local sum="" for i,v in tbl do for t,o in v do
if t~="narrator" then
sum=sum.."\n\n"..t..": "..o
else
sum=sum.."\n\n"..o
end end print(sum)
end
return sum
end
local function iterateoutputLLama(output)
local checkedinput
local loadoutput
local previnput
repeat
iterations+=1
previnput=HttpService:JSONDecode(output)[1].generated_text
local loadoutput =cm.TinyLlama(""..identity..timeod..insight..awarobserve..memory..". Parse dialogues with "..person..": and "..Playername..": ." ,quer,DecodeResponse(output))
if loadoutput~=nil then
checkedinput=HttpService:JSONDecode(loadoutput)[1].generated_text
if checkedinput then--only update output if valid
output=loadoutput
print(output)
else
break
end
else
break
end
until checkedinput==previnput or iterations>=3
return output
end
local output2=cm.TinyLlama(""..identity..timeod..insight..awarobserve..memory..". Parse dialogues with "..person..": and "..Playername..": ." ,quer,DecodeResponse(output))--recieve generated_text
if output2 then
iterations=0
local output3=iterateoutputLLama(output2)
if output3~=nil then
output=output3
else
output=output2
end
end
--local str=format_response(output[1].generated_text)
--print(str)
--local outputtabl=cm.extractDialogue(str)
-- Print the output
return output--[1] --parseConversation(str)
end
This code takes advantage of both models utilizing Zephyrs strong abilities to generate high quality output and Llamas 2048 context window to create longer responses.