Publish Sophon Realtime calling tool protocol and latency guide
This commit is contained in:
wuyang6
@@ -0,0 +1,208 @@
|
||||
export const builds = {
|
||||
flash: {
|
||||
title: "Flash",
|
||||
directory: "flash",
|
||||
modelDirectory: "Sophon-Realtime-32B-Flash",
|
||||
flow: ["用户语音", "共享 32B 理解与文本", "Flash 语音头", "24 kHz 语音"],
|
||||
detail:
|
||||
"声音侧以每 80 ms 帧一次前向为设计。作者报告单 H100 HTTP 首包中位数 0.31 s，实例约 55 GB 显存。",
|
||||
modes: ["http", "engine", "realtime", "live"],
|
||||
},
|
||||
tag: {
|
||||
title: "Tag",
|
||||
directory: "tag",
|
||||
modelDirectory: "Sophon-Realtime-32B-Tag",
|
||||
flow: [
|
||||
"用户语音",
|
||||
"共享 32B 与声音标签",
|
||||
"tag-tts 语音模型",
|
||||
"Flow + HiFT 语音",
|
||||
],
|
||||
detail:
|
||||
"模型写出笑声、叹气等声音标签，由专用语音模块表现。作者报告单 80 GB GPU、实例约 65 GB 显存。",
|
||||
modes: ["http", "engine", "realtime", "live"],
|
||||
},
|
||||
e2e: {
|
||||
title: "E2E 预览",
|
||||
directory: "",
|
||||
modelDirectory: "Sophon-Realtime-32B-E2E",
|
||||
flow: [
|
||||
"用户语音",
|
||||
"32B Thinker 与 Talker",
|
||||
"连续语音向量",
|
||||
"E2 解码与语音",
|
||||
],
|
||||
detail:
|
||||
"采用独立推理引擎与 /ws 网关，文档要求两张 80 GB GPU。当前作为研究预览，部署文档与模型目录版本需逐项固定。",
|
||||
modes: ["e2e"],
|
||||
},
|
||||
};
|
||||
|
||||
export const callModes = {
|
||||
http: {
|
||||
title: "HTTP 音频生成",
|
||||
kind: "POST",
|
||||
endpoint: "http://HOST:8000/v1/chat/completions",
|
||||
detail:
|
||||
"先用完整 WAV 验证语音生成。官方客户端先取 /v1/models，再流式解码文本与音频。",
|
||||
},
|
||||
engine: {
|
||||
title: "基础引擎双工",
|
||||
kind: "WebSocket",
|
||||
endpoint: "ws://HOST:8000/v1/realtime",
|
||||
detail:
|
||||
"持续推送音频，模型参与话轮决策。原生工具桥需要自行对接，默认 mock 与真实执行分别核验。",
|
||||
},
|
||||
realtime: {
|
||||
title: "网关函数调用",
|
||||
kind: "WebSocket",
|
||||
endpoint: "ws://HOST:8780/v1/realtime",
|
||||
detail:
|
||||
"启动 Live 网关后，使用其 Realtime 适配器接收 function_call 与回填结果。当前协议覆盖有边界。",
|
||||
},
|
||||
live: {
|
||||
title: "Live 客户端委派",
|
||||
kind: "WebSocket",
|
||||
endpoint: "ws://HOST:8780/v1/live/sessions",
|
||||
detail:
|
||||
"前台生成 delegate(task)，客户端接委派事件，并把后台结果经 commentary.append 返回。",
|
||||
},
|
||||
e2e: {
|
||||
title: "E2E 自有协议",
|
||||
kind: "WebSocket",
|
||||
endpoint: "ws://HOST:8000/ws",
|
||||
detail:
|
||||
"按 E2E 文档使用 config、二进制 PCM16、text、barge_in、done 等事件。",
|
||||
},
|
||||
};
|
||||
|
||||
export function makeRecipe(buildKey, modeKey) {
|
||||
const build = builds[buildKey];
|
||||
if (!build || !build.modes.includes(modeKey)) {
|
||||
throw new RangeError(
|
||||
"The selected build does not support this documented path.",
|
||||
);
|
||||
}
|
||||
if (modeKey === "e2e") {
|
||||
return {
|
||||
startup: `# 在已准备 Linux / CUDA / 权重的环境中
|
||||
SKIP_DOWNLOAD=1 MODEL_DIR=/path/to/${build.modelDirectory} bash scripts/install.sh
|
||||
SOPHON_MODEL_DIR=/path/to/${build.modelDirectory} THINKER_GPU=0 TALKER_GPU=1 bash scripts/start.sh`,
|
||||
request: `连接 ws://HOST:8000/ws 后先发送
|
||||
{"type":"config","lang":"zh","history":true}
|
||||
|
||||
随后持续发送 16 kHz、单声道 PCM16 二进制音频。
|
||||
接收 text / 二进制 24 kHz 音频 / barge_in / metrics / done。`,
|
||||
};
|
||||
}
|
||||
const startup = `# 在官方仓库 ${build.directory}/ 中，权重须预先准备
|
||||
SKIP_DOWNLOAD=1 bash install.sh
|
||||
MODEL_DIR=/path/to/${build.modelDirectory} GPU=0 PORT=8000 bash start.sh
|
||||
# 等待 READY 与预热完成`;
|
||||
if (modeKey === "http") {
|
||||
return {
|
||||
startup: `${startup}
|
||||
|
||||
python client_example.py --url http://HOST:8000 --wav question.wav --lang zh --voice chelsie`,
|
||||
request: `先从 GET /v1/models 取得模型 ID。
|
||||
POST /v1/chat/completions
|
||||
{
|
||||
"model": "<返回的模型 ID>",
|
||||
"stream": true,
|
||||
"modalities": ["text", "audio"],
|
||||
"language": "zh",
|
||||
"messages": [
|
||||
{"role":"system","content":"<config/system_prompt.json 的 zh 字段>"},
|
||||
{"role":"user","content":[
|
||||
{"type":"text","text":"<|audio|>"},
|
||||
{"type":"input_audio","input_audio":{"data":"<WAV_BASE64>","format":"wav"}}
|
||||
]}
|
||||
]
|
||||
}`,
|
||||
};
|
||||
}
|
||||
if (modeKey === "engine") {
|
||||
return {
|
||||
startup,
|
||||
request: `连接 ws://HOST:8000/v1/realtime
|
||||
{"type":"session.update","session":{"extra_body":{"auto_response":true,"language":"zh"}}}
|
||||
{"type":"input_audio_buffer.append","audio":"<持续发送的 PCM16_BASE64>"}
|
||||
|
||||
接真实工具时，额外配置 tools / tool_mode=client / bridge_key。
|
||||
工具桥默认在引擎本机 PORT+700，需配套执行与回填逻辑。`,
|
||||
};
|
||||
}
|
||||
const gatewayStartup = `${startup}
|
||||
|
||||
# 同一环境启动外层网关
|
||||
cd ../live
|
||||
SOPHON_LIVE_ASR=/path/to/Qwen3-ASR-0.6B-hf BUILD=${build.directory} PORT=8000 LIVE_PORT=8780 bash start_live.sh`;
|
||||
if (modeKey === "realtime") {
|
||||
return {
|
||||
startup: gatewayStartup,
|
||||
request: `连接 ws://HOST:8780/v1/realtime 后配置工具
|
||||
{"type":"session.update","session":{"instructions":"简短回答用户","tools":[
|
||||
{"type":"function","name":"get_weather","description":"查询城市天气",
|
||||
"parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}
|
||||
]}}
|
||||
|
||||
收到 function_call 后，由客户端执行真实函数，再回填
|
||||
{"type":"conversation.item.create","item":{"type":"function_call_output","call_id":"<返回的 call_id>","output":"<真实结果>"}}
|
||||
{"type":"response.create"}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
startup: gatewayStartup,
|
||||
request: `连接 ws://HOST:8780/v1/live/sessions
|
||||
{"type":"session.start","session":{"instructions":"用中文简短回答","delegation":{"type":"client"}}}
|
||||
{"type":"session.input_audio.append","audio":"<PCM16_BASE64>"}
|
||||
|
||||
接收 session.delegation.created，后台完成后回填
|
||||
{"type":"session.commentary.append","delegation_id":"<返回的委派 ID>","content":"<真实后台结果>"}`,
|
||||
};
|
||||
}
|
||||
|
||||
export const toolSteps = [
|
||||
{
|
||||
title: "提供函数定义",
|
||||
explanation:
|
||||
"调用方声明函数名、说明与参数 schema。运行时经 chat template 把这些定义提供给模型。",
|
||||
code: '{"type":"function","name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}',
|
||||
},
|
||||
{
|
||||
title: "模型生成调用",
|
||||
explanation:
|
||||
"模型可以先给一句承接语，再输出工具名与参数。下面的天气任务为合成示例。",
|
||||
code: '我帮你查一下。\n<tool_call>\n{"name":"get_weather","arguments":{"city":"上海"}}\n</tool_call>',
|
||||
},
|
||||
{
|
||||
title: "口语流与控制流分开",
|
||||
explanation:
|
||||
"cut_tool 保留调用标记前的口语。完整或尚未写完的工具标记会被拦住，避免进入语音合成。",
|
||||
code: '送往语音模块\n我帮你查一下。\n\n留给工具解析器\n<tool_call>{"name":"get_weather","arguments":{"city":"上海"}}</tool_call>',
|
||||
},
|
||||
{
|
||||
title: "解析与桥接",
|
||||
explanation:
|
||||
"Thinker 写完后，服务解析函数调用，发出带 call_id、name 和 arguments 的工具桥事件。此时承接语音可能仍在播放。",
|
||||
code: '{"type":"tool_call","call_id":"call_demo","name":"get_weather","arguments":{"city":"上海"}}',
|
||||
},
|
||||
{
|
||||
title: "客户端执行真实函数",
|
||||
explanation:
|
||||
"真实执行端取出函数和参数，调用业务系统。mock 模式则返回模拟值。两类证据必须分开记录。",
|
||||
code: 'result = await actual_weather_service(city="上海")\n# 记录真实调用 ID、参数、返回、耗时与最终状态',
|
||||
},
|
||||
{
|
||||
title: "回填结果",
|
||||
explanation:
|
||||
"结果经客户端工具桥，或网关的 function_call_output / commentary.append 返回。下方内容显式标为示意。",
|
||||
code: '{"type":"function_call_output","call_id":"call_demo","output":"示例结果：天气晴朗"}',
|
||||
},
|
||||
{
|
||||
title: "模型继续回应",
|
||||
explanation:
|
||||
"结果被加入工具历史，后续语音从该结果继续生成。正确选择函数、真实执行和正确告知各自评分。",
|
||||
code: "role: tool\n示例结果：天气晴朗\n\nassistant\n查到了，示例结果显示天气晴朗。",
|
||||
},
|
||||
];
|
||||
Reference in New Issue
Block a user