Gemini Live

const ws = new WebSocket('wss://llm.ai-nebula.com/ws/v1beta/models/gemini-live-2.5-flash-native-audio/liveStream', {
  headers: {
    'Authorization': 'Bearer sk-xxxx'
  }
});

const ws = new WebSocket('wss://llm.ai-nebula.com/ws/v1beta/models/gemini-live-2.5-flash-native-audio/liveStream', {
  headers: {
    'Authorization': 'Bearer sk-xxxx'
  }
});

ws.onopen = () => {
  console.log('WebSocket connected');
  
  // 发送 setup 消息
  ws.send(JSON.stringify({
    setup: {
      model: "gemini-live-2.5-flash-native-audio",
      generationConfig: {
        temperature: 0.7,
        responseModalities: ["AUDIO"]
      },
      systemInstruction: {
        parts: [
          { text: "You are a helpful assistant. Speak naturally and conversationally." }
        ]
      },
      speechConfig: {
        voiceConfig: {
          prebuiltVoiceConfig: {
            voiceName: "Puck"
          }
        }
      }
    }
  }));
};

ws.onmessage = (event) => {
  const message = JSON.parse(event.data);
  console.log('Received:', message);
  
  if (message.serverContent) {
    // 处理输出转录（音频转文本）
    if (message.serverContent.outputTranscription) {
      const text = message.serverContent.outputTranscription.text;
      if (text) {
        console.log('[转录]', text);
      }
    }
    
    if (message.serverContent.modelTurn) {
      // 处理模型输出
      message.serverContent.modelTurn.parts.forEach(part => {
        if (part.text) {
          console.log('Text:', part.text);
        }
        if (part.inlineData && part.inlineData.mimeType === "audio/pcm") {
          // 处理音频数据
          const audioData = part.inlineData.data;
          // audioData 是 base64 编码的 PCM 音频
        }
      });
    }
    if (message.serverContent.turnComplete) {
      console.log('Turn complete');
    }
  }
  
  if (message.setupComplete) {
    console.log('Setup complete');
  }
};

// 发送实时音频输入
function sendRealtimeAudio(audioBuffer) {
  const base64Audio = btoa(
    String.fromCharCode(...new Uint8Array(audioBuffer))
  );
  
  ws.send(JSON.stringify({
    realtimeInput: {
      mediaChunks: [
        {
          mimeType: "audio/pcm;rate=16000",
          data: base64Audio
        }
      ]
    }
  }));
}

// 在 setupComplete 后发送文本消息
function sendText(text) {
  // Nebula 已测试的 Gemini Live 路由支持 setup 后直接发送纯文本。
  ws.send(text);
}

import websocket
import json
import base64
import threading

def on_message(ws, message):
    data = json.loads(message)
    print(f"Received: {data}")
    
    # 处理输出转录
    if "serverContent" in data:
        server_content = data["serverContent"]
        
        if "outputTranscription" in server_content:
            transcription = server_content["outputTranscription"]
            text = transcription.get("text", "")
            if text:
                print(f"[转录] {text}")
        
        if "modelTurn" in server_content:
            model_turn = server_content["modelTurn"]
            if "parts" in model_turn:
                for part in model_turn["parts"]:
                    if "text" in part:
                        print(f"Text: {part['text']}")
                    elif "inlineData" in part:
                        inline_data = part["inlineData"]
                        if inline_data.get("mimeType") == "audio/pcm":
                            audio_b64 = inline_data.get("data", "")
                            if audio_b64:
                                audio_data = base64.b64decode(audio_b64)
                                # 处理音频数据

def on_error(ws, error):
    print(f"Error: {error}")

def on_close(ws, close_status_code, close_msg):
    print("Connection closed")

def on_open(ws):
    print("WebSocket connected")
    
    # 发送 setup 消息
    setup_message = {
        "setup": {
            "model": "gemini-live-2.5-flash-native-audio",
            "generationConfig": {
                "temperature": 0.7,
                "responseModalities": ["AUDIO"]
            },
            "systemInstruction": {
                "parts": [
                    {"text": "You are a helpful assistant."}
                ]
            },
            "speechConfig": {
                "voiceConfig": {
                    "prebuiltVoiceConfig": {
                        "voiceName": "Puck"
                    }
                }
            }
        }
    }
    ws.send(json.dumps(setup_message))

# 连接 WebSocket
ws_url = "wss://llm.ai-nebula.com/ws/v1beta/models/gemini-live-2.5-flash-native-audio/liveStream"
ws = websocket.WebSocketApp(
    ws_url,
    header={"Authorization": "Bearer sk-xxxx"},
    on_open=on_open,
    on_message=on_message,
    on_error=on_error,
    on_close=on_close
)

ws.run_forever()

{
  "setup": {
    "model": "gemini-live-2.5-flash-native-audio",
    "generationConfig": {
      "temperature": 0.7,
      "responseModalities": ["AUDIO"],
      "speechConfig": {
        "voiceConfig": {
          "prebuiltVoiceConfig": {
            "voiceName": "Zephyr"
          }
        },
        "languageCode": "zh-CN"
      }
    },
    "systemInstruction": {
      "parts": [
        {"text": "你是一个友好的助手，请用自然、对话式的方式回答问题。"}
      ]
    }
  }
}

{
  "setup": {
    "model": "gemini-live-2.5-flash-native-audio",
    "generationConfig": {
      "temperature": 0.7,
      "responseModalities": ["AUDIO", "TEXT"],
      "speechConfig": {
        "voiceConfig": {
          "prebuiltVoiceConfig": {
            "voiceName": "Zephyr"
          }
        },
        "languageCode": "zh-CN"
      }
    },
    "systemInstruction": {
      "parts": [
        {"text": "你是一个友好的助手，请用自然、对话式的方式回答问题。"}
      ]
    },
    "tools": {
      "googleSearch": {}
    },
    "proactivity": {
      "proactiveAudio": false,
      "empatheticMode": true
    },
    "outputAudioTranscription": {},
    "realtimeInputConfig": {
      "automaticActivityDetection": {
        "disabled": false,
        "startOfSpeechSensitivity": "START_SENSITIVITY_LOW",
        "endOfSpeechSensitivity": "END_SENSITIVITY_HIGH",
        "prefixPaddingMs": 0,
        "silenceDurationMs": 0
      }
    }
  }
}