GPT-4o in Unity

API Key

First we need an OpenAI API key.

• Go to https://platform.openai.com/api-keys and create a key (if you don’t have an OpenAI account, sign up first)

• Copy the key, we’re going to need it in a moment.

 

System Environment Variable

Whoever is in posession of our secret key can call the API and thus spend money on our OpenAI account, so we want to keep it safe on our system. To make sure we don’t accidentally include it in our source code, we’ll store it in a System Environment Variable. This also allows us to reuse it across multiple projects with ease.

var secretKey = Environment.GetEnvironmentVariable( "OPENAI_API_KEY", EnvironmentVariableTarget.User );

Unity

Now we’re going to create a script that talks to the OpenAI API and

• Open a Unity Project or create a new one.

• Install the NewtonSoft JSON package
• Window > Package Manager
• Click the + and choose Install package by name...
• Enter com.unity.nuget.newtonsoft-json and watch it ⬇️ install.

• Create a new MonoBehaviour script called OpenAIConnection.cs and paste in the following code:

using System;
using UnityEngine;
using UnityEngine.Networking;
using Newtonsoft.Json;
using System.Collections.Generic;
using System.Text;
using System.Collections;

public class OpenAIConnection : MonoBehaviour {

		private string m_OpenAIEndpoint = "https://api.openai.com/v1/chat/completions";
    [SerializeField] private string m_Model = "gpt-4o";
    [SerializeField] private List<Message> m_Messages = new() { new Message { role = "system", content = "You are a helpful assistant." } };
    private List<Message> m_InitialMessages = new();
    
    [Serializable]
    public class Message {
        public string role;
        public string content;
    }

    [Serializable]
    public class ChatRequest {
        public string model;
        public List<Message> messages;
    }

    [Serializable]
    public class ChatResponse {
        public List<Choice> choices;
    }

    [Serializable]
    public class Choice {
        public Message message;
    }

    void Start() {
        m_InitialMessages.AddRange( m_Messages ); // remember initial messages for later use in Clear 
        //Send( "Tell me a joke about modular synthesizers.", OnResponseReceived ); //Test
    }

    private void OnResponseReceived( bool success, string message ) {
        if(success) Debug.Log( message );
        else Debug.LogError( message );
    }

    public void Send( string message, Action<bool, string> callback ) {
        Send( new Message() { role = "user", content = message}, callback );
    }
    public void Send( Message message, Action<bool, string> callback ) {
        m_Messages.Add( message );
        StartCoroutine( SendRequest( m_Messages, callback ) );
    }

    public void Clear() {
        m_Messages.Clear();
        m_Messages.AddRange( m_InitialMessages );
    }
    
    private IEnumerator SendRequest( List<Message> messages, Action<bool, string> callback ) {
        ChatRequest chatRequest = new ChatRequest {
            model = m_Model,
            messages = messages
        };

        string jsonData = JsonConvert.SerializeObject( chatRequest );
        byte[] jsonBytes = Encoding.UTF8.GetBytes( jsonData );

        using( var request = new UnityWebRequest( m_OpenAIEndpoint, "POST" ) ) {
            request.uploadHandler = new UploadHandlerRaw( jsonBytes );
            request.downloadHandler = new DownloadHandlerBuffer();
            var secretKey = Environment.GetEnvironmentVariable( "OPENAI_API_KEY", EnvironmentVariableTarget.User );
            request.SetRequestHeader( "Authorization", "Bearer " + secretKey );
            request.SetRequestHeader( "Content-Type", "application/json" );

            yield return request.SendWebRequest();

            if( request.result != UnityWebRequest.Result.Success ) {
                callback?.Invoke( false, request.error );
            }
            else {
                string responseText = request.downloadHandler.text;
                ChatResponse chatResponse = JsonConvert.DeserializeObject<ChatResponse>( responseText );
                
                if( chatResponse.choices != null && chatResponse.choices.Count > 0 ) {
                    m_Messages.Add( chatResponse.choices[0].message );
                    callback?.Invoke( true, chatResponse.choices[0].message.content );
                }
                else {
                    callback?.Invoke( false, "No choices received" );
                }
            }
        }
    }
}

Add it to a GameObject and you’re ready to go!
To test it without any further work, you can uncomment the Send line in the Start method and hit ▶️ Play.

How to use it

If you look at the OpenAIConnection component you can see that it comes preconfigured like this:



Model - set the model you want to use (pick one from here: 📋https://platform.openai.com/docs/models/gpt-4o)
Messages - configure what starting instructions the model receives. While in play mode you can see the message history build up in here.

The entire message history is sent along with every new request!
💬 Sending Requests - call Send( message, callback ) on OpenAIConnection to add a message to the list of messages and send it to the API. The second required argument is a callback function that will be invoked once the response has arrived.
🧽 Clear - call Clear() to restore the message history to its initial state.

Done!


And there you have it! GPT-4o answers your requests in Unity!

Bonus: Chat UI

If you want to build a quick chat UI, grab this script and add some UGUI elements: An InputField, a Button and a Text, then link it all up and start chatting.

using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.UI;

public class ChatUI : MonoBehaviour {

	[SerializeField] private TMP_InputField m_InputField;
	[SerializeField] private TextMeshProUGUI m_Text;
	[SerializeField] private Button m_Button;
	[SerializeField] private ScrollRect m_ScrollRect;
	
	[SerializeField] private OpenAIConnection m_OpenAIConnection;

	void Start() {
		m_Button.onClick.AddListener( OnButtonClick );
		m_InputField.onSubmit.AddListener( delegate { OnButtonClick(); } );
	}

	private void OnButtonClick() {
		m_Text.text += $"User: {m_InputField.text}\n\n";
		m_OpenAIConnection.Send( m_InputField.text, Callback );
		m_InputField.text = "";
		m_Button.enabled = false;
	}

	private void Callback( bool success, string message ) {
		m_Text.text += $"<color=#8888FF>Assistant: {message}</color>\n\n";
		m_Button.enabled = true;
		
		StartCoroutine( ApplyScrollPosition( m_ScrollRect, 1f ) );
	}
	
	IEnumerator ApplyScrollPosition( ScrollRect sr, float verticalPos ) {
		yield return new WaitForEndOfFrame( );
		sr.verticalNormalizedPosition = verticalPos;
		LayoutRebuilder.ForceRebuildLayoutImmediate( (RectTransform)sr.transform );
	}
}

I made mine look like this: