Realtime API

WSS wss://llm.ai-nebula.com/v1/realtime?model={model}

{
  "event_id": "evt_001",
  "type": "session.update",
  "session": {
    "modalities": ["text", "audio"],
    "instructions": "You are a friendly assistant",
    "voice": "alloy",
    "temperature": 0.8,
    "input_audio_format": "pcm16",
    "output_audio_format": "pcm16"
  }
}

{
  "event_id": "evt_001",
  "type": "session.update",
  "session": {
    "type": "realtime",
    "instructions": "You are a concise assistant"
  }
}

{ "event_id": "evt_002", "type": "conversation.item.create", "item": { "type": "message", "role": "user", "content": [{ "type": "input_text", "text": "Reply with exactly: ok" }] } }

{ "event_id": "evt_003", "type": "response.create" }

{
  "event_id": "evt_002",
  "type": "conversation.item.create",
  "item": {
    "type": "message",
    "role": "user",
    "content": [
      { "type": "input_text", "text": "Hello, please briefly introduce yourself." }
    ]
  }
}

// 1. Push audio data (can be called multiple times)
{
  "event_id": "evt_003",
  "type": "input_audio_buffer.append",
  "audio": "base64_encoded_audio_data..."
}

// 2. Commit audio buffer
{
  "event_id": "evt_004",
  "type": "input_audio_buffer.commit"
}

// 3. Create conversation item
{
  "event_id": "evt_005",
  "type": "conversation.item.create",
  "item": {
    "type": "message",
    "role": "user",
    "content": [
      { "type": "input_audio", "audio": "" }
    ]
  }
}

{
  "event_id": "evt_006",
  "type": "response.create"
}

import json
import websocket
import threading

API_BASE = "wss://llm.ai-nebula.com"
API_KEY = "sk-xxxxxxxxxx"
MODEL = "gpt-realtime"

def on_message(ws, message):
    """Handle server-returned messages"""
    event = json.loads(message)
    event_type = event.get("type")
    
    if event_type == "session.created":
        print(f"Session created: {event.get('session', {}).get('id')}")
    elif event_type == "response.text.delta":
        # Stream text output
        delta = event.get("delta", "")
        print(delta, end="", flush=True)
    elif event_type == "response.done":
        # Response completed, show usage statistics
        usage = event.get("response", {}).get("usage", {})
        print(f"\n\nToken usage: {usage.get('total_tokens', 0)}")
    elif event_type == "error":
        error = event.get("error", {})
        print(f"\nError: {error.get('message', 'Unknown error')}")

def on_error(ws, error):
    print(f"WebSocket error: {error}")

def on_close(ws, close_status_code, close_msg):
    print("Connection closed")

def on_open(ws):
    """Send initial messages after connection is established"""
    # 1. Configure session
    session_config = {
        "event_id": "evt_001",
        "type": "session.update",
        "session": {
            "modalities": ["text"],
            "instructions": "You are a concise assistant",
            "temperature": 0.8
        }
    }
    ws.send(json.dumps(session_config, ensure_ascii=False))
    
    # 2. Send user message
    user_message = {
        "event_id": "evt_002",
        "type": "conversation.item.create",
        "item": {
            "type": "message",
            "role": "user",
            "content": [
                {"type": "input_text", "text": "Introduce Nebula in one sentence."}
            ]
        }
    }
    ws.send(json.dumps(user_message, ensure_ascii=False))
    
    # 3. Request response generation
    response_create = {
        "event_id": "evt_003",
        "type": "response.create"
    }
    ws.send(json.dumps(response_create, ensure_ascii=False))

# Establish WebSocket connection
ws = websocket.WebSocketApp(
    f"{API_BASE}/v1/realtime?model={MODEL}",
    header={"Authorization": f"Bearer {API_KEY}"},
    on_message=on_message,
    on_error=on_error,
    on_close=on_close,
    on_open=on_open
)

# Run WebSocket (blocking)
ws.run_forever()

const API_BASE = 'wss://llm.ai-nebula.com';
const API_KEY = 'sk-xxxxxxxxxx';
const MODEL = 'gpt-realtime';

const ws = new WebSocket(`${API_BASE}/v1/realtime?model=${MODEL}`, [], {
  headers: {
    'Authorization': `Bearer ${API_KEY}`
  }
});

ws.onopen = () => {
  console.log('WebSocket connection established');
  
  // 1. Configure session
  ws.send(JSON.stringify({
    event_id: 'evt_001',
    type: 'session.update',
    session: {
      modalities: ['text'],
      instructions: 'You are a concise assistant',
      temperature: 0.8
    }
  }));
  
  // 2. Send user message
  ws.send(JSON.stringify({
    event_id: 'evt_002',
    type: 'conversation.item.create',
    item: {
      type: 'message',
      role: 'user',
      content: [
        { type: 'input_text', text: 'Introduce Nebula in one sentence.' }
      ]
    }
  }));
  
  // 3. Request response generation
  ws.send(JSON.stringify({
    event_id: 'evt_003',
    type: 'response.create'
  }));
};

ws.onmessage = (event) => {
  const message = JSON.parse(event.data);
  const eventType = message.type;
  
  switch (eventType) {
    case 'session.created':
      console.log('Session created:', message.session?.id);
      break;
    case 'response.text.delta':
      // Stream text output
      process.stdout.write(message.delta || '');
      break;
    case 'response.done':
      // Response completed
      const usage = message.response?.usage;
      console.log('\n\nToken usage:', usage?.total_tokens || 0);
      break;
    case 'error':
      console.error('Error:', message.error?.message);
      break;
  }
};

ws.onerror = (error) => {
  console.error('WebSocket error:', error);
};

ws.onclose = () => {
  console.log('Connection closed');
};

{
  "type": "error",
  "error": {
    "type": "invalid_request_error",
    "message": "Error description",
    "code": "error_code"
  }
}