Publish voice duplex architecture latency training and evaluation research
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
export function queueSteps(chunkMs, computeMs, count = 8) {
|
||||
if (
|
||||
!Number.isFinite(chunkMs) ||
|
||||
!Number.isFinite(computeMs) ||
|
||||
chunkMs <= 0 ||
|
||||
computeMs <= 0
|
||||
) {
|
||||
throw new RangeError("Time values must be finite and positive.");
|
||||
}
|
||||
let previousEnd = 0;
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const arrival = (index + 1) * chunkMs;
|
||||
const start = Math.max(arrival, previousEnd);
|
||||
const end = start + computeMs;
|
||||
previousEnd = end;
|
||||
return { index: index + 1, arrival, start, end, wait: start - arrival };
|
||||
});
|
||||
}
|
||||
|
||||
export const architectures = {
|
||||
cascade: {
|
||||
nodes: [
|
||||
["用户语音", "声音输入", "input"],
|
||||
["ASR", "识别成文本"],
|
||||
["文本 LLM", "生成回答"],
|
||||
["TTS", "合成声音"],
|
||||
["系统语音", "播放输出", "output"],
|
||||
],
|
||||
detail:
|
||||
"<p><strong>两个独立的文本接口。</strong>ASR 把用户语音交给文本模型,TTS 再把回答读出。模块可以独立替换。仅传文字时,输入声学信息会在接口处被压缩。</p><p>持续双工需要在这些模块之间保持新输入、话轮状态、取消与播放进度的一致。</p>",
|
||||
},
|
||||
two: {
|
||||
nodes: [
|
||||
["用户语音", "声音输入", "input"],
|
||||
["语音理解 LLM", "直接读取音频特征"],
|
||||
["回答文本", "内容接口"],
|
||||
["TTS", "合成声音"],
|
||||
["系统语音", "播放输出", "output"],
|
||||
],
|
||||
detail:
|
||||
"<p><strong>输入端直接理解声音。</strong>模型接收音频特征并生成回答文字,之后由独立 TTS 合成。输入端可以保留语气等线索,输出端仍通过文本接口组织。</p><p>本页以 Ultravox 核心加外接 TTS 作为这一工作定义的例子。</p>",
|
||||
},
|
||||
native: {
|
||||
nodes: [
|
||||
["用户语音", "声音输入", "input"],
|
||||
["语音表示", "编码或特征"],
|
||||
["联合交互模型", "声音与内容建模"],
|
||||
["语音解码", "还原波形"],
|
||||
["系统语音", "播放输出", "output"],
|
||||
],
|
||||
detail:
|
||||
"<p><strong>语音直接参与内容与声学生成。</strong>编码器、音频 token、文字辅助流和波形解码器可以共同存在,具体组织依模型而异。</p><p>持续听说依赖输入更新方式、因果训练与实时服务。原生语音接口与全双工行为分别核验。</p>",
|
||||
},
|
||||
};
|
||||
|
||||
export const benchmarks = [
|
||||
{ name: "级联管线", first: 8.78, answer: 10.12 },
|
||||
{ name: "Ultravox 0.7", first: 3.88, answer: 8.4 },
|
||||
{ name: "GPT-Realtime 1.5", first: 6.36, answer: 6.89 },
|
||||
{ name: "Gemini Live 3.1", first: 3.95, answer: 4.25 },
|
||||
];
|
||||
Reference in New Issue
Block a user