sophon-realtime-guide/model.js

export const builds = {
  flash: {
    title: "Flash",
    directory: "flash",
    modelDirectory: "Sophon-Realtime-32B-Flash",
    flow: ["用户语音", "共享 32B 理解与文本", "Flash 语音头", "24 kHz 语音"],
    detail:
      "声音侧以每 80 ms 帧一次前向为设计。作者报告单 H100 HTTP 首包中位数 0.31 s，实例约 55 GB 显存。",
    modes: ["http", "engine", "realtime", "live"],
  },
  tag: {
    title: "Tag",
    directory: "tag",
    modelDirectory: "Sophon-Realtime-32B-Tag",
    flow: [
      "用户语音",
      "共享 32B 与声音标签",
      "tag-tts 语音模型",
      "Flow + HiFT 语音",
    ],
    detail:
      "模型写出笑声、叹气等声音标签，由专用语音模块表现。作者报告单 80 GB GPU、实例约 65 GB 显存。",
    modes: ["http", "engine", "realtime", "live"],
  },
  e2e: {
    title: "E2E 预览",
    directory: "",
    modelDirectory: "Sophon-Realtime-32B-E2E",
    flow: [
      "用户语音",
      "32B Thinker 与 Talker",
      "连续语音向量",
      "E2 解码与语音",
    ],
    detail:
      "采用独立推理引擎与 /ws 网关，文档要求两张 80 GB GPU。当前作为研究预览，部署文档与模型目录版本需逐项固定。",
    modes: ["e2e"],
  },
};

export const callModes = {
  http: {
    title: "HTTP 音频生成",
    kind: "POST",
    endpoint: "http://HOST:8000/v1/chat/completions",
    detail:
      "先用完整 WAV 验证语音生成。官方客户端先取 /v1/models，再流式解码文本与音频。",
  },
  engine: {
    title: "基础引擎双工",
    kind: "WebSocket",
    endpoint: "ws://HOST:8000/v1/realtime",
    detail:
      "持续推送音频，模型参与话轮决策。原生工具桥需要自行对接，默认 mock 与真实执行分别核验。",
  },
  realtime: {
    title: "网关函数调用",
    kind: "WebSocket",
    endpoint: "ws://HOST:8780/v1/realtime",
    detail:
      "启动 Live 网关后，使用其 Realtime 适配器接收 function_call 与回填结果。当前协议覆盖有边界。",
  },
  live: {
    title: "Live 客户端委派",
    kind: "WebSocket",
    endpoint: "ws://HOST:8780/v1/live/sessions",
    detail:
      "前台生成 delegate(task)，客户端接委派事件，并把后台结果经 commentary.append 返回。",
  },
  e2e: {
    title: "E2E 自有协议",
    kind: "WebSocket",
    endpoint: "ws://HOST:8000/ws",
    detail:
      "按 E2E 文档使用 config、二进制 PCM16、text、barge_in、done 等事件。",
  },
};

export function makeRecipe(buildKey, modeKey) {
  const build = builds[buildKey];
  if (!build || !build.modes.includes(modeKey)) {
    throw new RangeError(
      "The selected build does not support this documented path.",
    );
  }
  if (modeKey === "e2e") {
    return {
      startup: `# 在已准备 Linux / CUDA / 权重的环境中
SKIP_DOWNLOAD=1 MODEL_DIR=/path/to/${build.modelDirectory} bash scripts/install.sh
SOPHON_MODEL_DIR=/path/to/${build.modelDirectory} THINKER_GPU=0 TALKER_GPU=1 bash scripts/start.sh`,
      request: `连接 ws://HOST:8000/ws 后先发送
{"type":"config","lang":"zh","history":true}

随后持续发送 16 kHz、单声道 PCM16 二进制音频。
接收 text / 二进制 24 kHz 音频 / barge_in / metrics / done。`,
    };
  }
  const startup = `# 在官方仓库 ${build.directory}/ 中，权重须预先准备
SKIP_DOWNLOAD=1 bash install.sh
MODEL_DIR=/path/to/${build.modelDirectory} GPU=0 PORT=8000 bash start.sh
# 等待 READY 与预热完成`;
  if (modeKey === "http") {
    return {
      startup: `${startup}

python client_example.py --url http://HOST:8000 --wav question.wav --lang zh --voice chelsie`,
      request: `先从 GET /v1/models 取得模型 ID。
POST /v1/chat/completions
{
  "model": "<返回的模型 ID>",
  "stream": true,
  "modalities": ["text", "audio"],
  "language": "zh",
  "messages": [
    {"role":"system","content":"<config/system_prompt.json 的 zh 字段>"},
    {"role":"user","content":[
      {"type":"text","text":"<|audio|>"},
      {"type":"input_audio","input_audio":{"data":"<WAV_BASE64>","format":"wav"}}
    ]}
  ]
}`,
    };
  }
  if (modeKey === "engine") {
    return {
      startup,
      request: `连接 ws://HOST:8000/v1/realtime
{"type":"session.update","session":{"extra_body":{"auto_response":true,"language":"zh"}}}
{"type":"input_audio_buffer.append","audio":"<持续发送的 PCM16_BASE64>"}

接真实工具时，额外配置 tools / tool_mode=client / bridge_key。
工具桥默认在引擎本机 PORT+700，需配套执行与回填逻辑。`,
    };
  }
  const gatewayStartup = `${startup}

# 同一环境启动外层网关
cd ../live
SOPHON_LIVE_ASR=/path/to/Qwen3-ASR-0.6B-hf BUILD=${build.directory} PORT=8000 LIVE_PORT=8780 bash start_live.sh`;
  if (modeKey === "realtime") {
    return {
      startup: gatewayStartup,
      request: `连接 ws://HOST:8780/v1/realtime 后配置工具
{"type":"session.update","session":{"instructions":"简短回答用户","tools":[
  {"type":"function","name":"get_weather","description":"查询城市天气",
   "parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}
]}}

收到 function_call 后，由客户端执行真实函数，再回填
{"type":"conversation.item.create","item":{"type":"function_call_output","call_id":"<返回的 call_id>","output":"<真实结果>"}}
{"type":"response.create"}`,
    };
  }
  return {
    startup: gatewayStartup,
    request: `连接 ws://HOST:8780/v1/live/sessions
{"type":"session.start","session":{"instructions":"用中文简短回答","delegation":{"type":"client"}}}
{"type":"session.input_audio.append","audio":"<PCM16_BASE64>"}

接收 session.delegation.created，后台完成后回填
{"type":"session.commentary.append","delegation_id":"<返回的委派 ID>","content":"<真实后台结果>"}`,
  };
}

export const toolSteps = [
  {
    title: "提供函数定义",
    explanation:
      "调用方声明函数名、说明与参数 schema。运行时经 chat template 把这些定义提供给模型。",
    code: '{"type":"function","name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}',
  },
  {
    title: "模型生成调用",
    explanation:
      "模型可以先给一句承接语，再输出工具名与参数。下面的天气任务为合成示例。",
    code: '我帮你查一下。\n<tool_call>\n{"name":"get_weather","arguments":{"city":"上海"}}\n</tool_call>',
  },
  {
    title: "口语流与控制流分开",
    explanation:
      "cut_tool 保留调用标记前的口语。完整或尚未写完的工具标记会被拦住，避免进入语音合成。",
    code: '送往语音模块\n我帮你查一下。\n\n留给工具解析器\n<tool_call>{"name":"get_weather","arguments":{"city":"上海"}}</tool_call>',
  },
  {
    title: "解析与桥接",
    explanation:
      "Thinker 写完后，服务解析函数调用，发出带 call_id、name 和 arguments 的工具桥事件。此时承接语音可能仍在播放。",
    code: '{"type":"tool_call","call_id":"call_demo","name":"get_weather","arguments":{"city":"上海"}}',
  },
  {
    title: "客户端执行真实函数",
    explanation:
      "真实执行端取出函数和参数，调用业务系统。mock 模式则返回模拟值。两类证据必须分开记录。",
    code: 'result = await actual_weather_service(city="上海")\n# 记录真实调用 ID、参数、返回、耗时与最终状态',
  },
  {
    title: "回填结果",
    explanation:
      "结果经客户端工具桥，或网关的 function_call_output / commentary.append 返回。下方内容显式标为示意。",
    code: '{"type":"function_call_output","call_id":"call_demo","output":"示例结果：天气晴朗"}',
  },
  {
    title: "模型继续回应",
    explanation:
      "结果被加入工具历史，后续语音从该结果继续生成。正确选择函数、真实执行和正确告知各自评分。",
    code: "role: tool\n示例结果：天气晴朗\n\nassistant\n查到了，示例结果显示天气晴朗。",
  },
];