Format the estimator and research UI source for review
This commit is contained in:
wuyang6
+282
-72
@@ -1,90 +1,300 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {simulateGraph,estimateNode,validateGraph,validateDocument,DEFAULT_GLOBALS,autoLayout} from '../src/engine.js';
|
||||
import {makeDocument,nodeData,PRESETS} from '../src/presets.js';
|
||||
import {RECORDS,SOURCES} from '../src/catalog.js';
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
simulateGraph,
|
||||
estimateNode,
|
||||
validateGraph,
|
||||
validateDocument,
|
||||
DEFAULT_GLOBALS,
|
||||
autoLayout,
|
||||
} from "../src/engine.js";
|
||||
import { makeDocument, nodeData, PRESETS } from "../src/presets.js";
|
||||
import { RECORDS, SOURCES } from "../src/catalog.js";
|
||||
|
||||
const g={...DEFAULT_GLOBALS,networkMs:0,vadMs:0,queueMs:0,speechSeconds:1};
|
||||
const node=(id,kind,extra={})=>({id,type:'stage',position:{x:0,y:0},data:{...nodeData(kind,id),...extra}});
|
||||
const edge=(source,target,release='complete',extra={})=>({id:`${source}-${target}`,source,target,data:{release,...extra}});
|
||||
const near=(a,b)=>assert.ok(Math.abs(a-b)<.00001,`${a} != ${b}`);
|
||||
const g = {
|
||||
...DEFAULT_GLOBALS,
|
||||
networkMs: 0,
|
||||
vadMs: 0,
|
||||
queueMs: 0,
|
||||
speechSeconds: 1,
|
||||
};
|
||||
const node = (id, kind, extra = {}) => ({
|
||||
id,
|
||||
type: "stage",
|
||||
position: { x: 0, y: 0 },
|
||||
data: { ...nodeData(kind, id), ...extra },
|
||||
});
|
||||
const edge = (source, target, release = "complete", extra = {}) => ({
|
||||
id: `${source}-${target}`,
|
||||
source,
|
||||
target,
|
||||
data: { release, ...extra },
|
||||
});
|
||||
const near = (a, b) => assert.ok(Math.abs(a - b) < 0.00001, `${a} != ${b}`);
|
||||
|
||||
test('serial stages add their service times',()=>{
|
||||
const n=[node('a','program',{durationMs:100}),node('b','program',{durationMs:200}),node('out','output',{durationMs:0,outputType:'text'})];
|
||||
const s=simulateGraph(n,[edge('a','b'),edge('b','out')],g);
|
||||
near(s.first,300);near(s.complete,300);
|
||||
test("serial stages add their service times", () => {
|
||||
const n = [
|
||||
node("a", "program", { durationMs: 100 }),
|
||||
node("b", "program", { durationMs: 200 }),
|
||||
node("out", "output", { durationMs: 0, outputType: "text" }),
|
||||
];
|
||||
const s = simulateGraph(n, [edge("a", "b"), edge("b", "out")], g);
|
||||
near(s.first, 300);
|
||||
near(s.complete, 300);
|
||||
});
|
||||
test('parallel all-join waits for longest branch rather than summing branches',()=>{
|
||||
const n=[node('in','input'),node('a','program',{durationMs:100}),node('b','program',{durationMs:300}),node('join','merge',{durationMs:20}),node('out','output',{durationMs:40,outputType:'text'})];
|
||||
const e=[edge('in','a'),edge('in','b'),edge('a','join'),edge('b','join'),edge('join','out')];
|
||||
const s=simulateGraph(n,e,g);near(s.first,360);assert.ok(s.criticalNodes.has('b'));assert.ok(!s.criticalNodes.has('a'));
|
||||
const any=n.map(x=>x.id==='join'?{...x,data:{...x.data,join:'any'}}:x);
|
||||
near(simulateGraph(any,e,g).first,160);
|
||||
test("parallel all-join waits for longest branch rather than summing branches", () => {
|
||||
const n = [
|
||||
node("in", "input"),
|
||||
node("a", "program", { durationMs: 100 }),
|
||||
node("b", "program", { durationMs: 300 }),
|
||||
node("join", "merge", { durationMs: 20 }),
|
||||
node("out", "output", { durationMs: 40, outputType: "text" }),
|
||||
];
|
||||
const e = [
|
||||
edge("in", "a"),
|
||||
edge("in", "b"),
|
||||
edge("a", "join"),
|
||||
edge("b", "join"),
|
||||
edge("join", "out"),
|
||||
];
|
||||
const s = simulateGraph(n, e, g);
|
||||
near(s.first, 360);
|
||||
assert.ok(s.criticalNodes.has("b"));
|
||||
assert.ok(!s.criticalNodes.has("a"));
|
||||
const any = n.map((x) =>
|
||||
x.id === "join" ? { ...x, data: { ...x.data, join: "any" } } : x,
|
||||
);
|
||||
near(simulateGraph(any, e, g).first, 160);
|
||||
});
|
||||
test('streaming starts early, completion remains bounded by final upstream data',()=>{
|
||||
const n=[node('llm','llm',{mode:'custom',customFirstMs:100,customTps:100,outputTokens:101}),node('tts','tts',{mode:'custom',customFirstMs:50,customRtf:.1}),node('out','output',{durationMs:0})];
|
||||
const e=[edge('llm','tts','first'),edge('tts','out','first')],s=simulateGraph(n,e,g);
|
||||
near(s.first,150);near(s.results.llm.end,1100);assert.ok(s.results.tts.end>=s.results.llm.end);assert.ok(s.complete>=s.first+1000);
|
||||
const full=simulateGraph(n,[edge('llm','tts'),edge('tts','out','first')],g);
|
||||
assert.ok(full.first>s.first);
|
||||
test("streaming starts early, completion remains bounded by final upstream data", () => {
|
||||
const n = [
|
||||
node("llm", "llm", {
|
||||
mode: "custom",
|
||||
customFirstMs: 100,
|
||||
customTps: 100,
|
||||
outputTokens: 101,
|
||||
}),
|
||||
node("tts", "tts", { mode: "custom", customFirstMs: 50, customRtf: 0.1 }),
|
||||
node("out", "output", { durationMs: 0 }),
|
||||
];
|
||||
const e = [edge("llm", "tts", "first"), edge("tts", "out", "first")],
|
||||
s = simulateGraph(n, e, g);
|
||||
near(s.first, 150);
|
||||
near(s.results.llm.end, 1100);
|
||||
assert.ok(s.results.tts.end >= s.results.llm.end);
|
||||
assert.ok(s.complete >= s.first + 1000);
|
||||
const full = simulateGraph(
|
||||
n,
|
||||
[edge("llm", "tts"), edge("tts", "out", "first")],
|
||||
g,
|
||||
);
|
||||
assert.ok(full.first > s.first);
|
||||
});
|
||||
test('token gate adds k-1 decoding intervals',()=>{
|
||||
const n=[node('llm','llm',{mode:'custom',customFirstMs:100,customTps:100,outputTokens:101}),node('out','output',{durationMs:0,outputType:'text'})];
|
||||
near(simulateGraph(n,[edge('llm','out','chunk',{tokens:8})],g).first,170);
|
||||
test("token gate adds k-1 decoding intervals", () => {
|
||||
const n = [
|
||||
node("llm", "llm", {
|
||||
mode: "custom",
|
||||
customFirstMs: 100,
|
||||
customTps: 100,
|
||||
outputTokens: 101,
|
||||
}),
|
||||
node("out", "output", { durationMs: 0, outputType: "text" }),
|
||||
];
|
||||
near(
|
||||
simulateGraph(n, [edge("llm", "out", "chunk", { tokens: 8 })], g).first,
|
||||
170,
|
||||
);
|
||||
});
|
||||
test('loop maximum, inter-step tools, final-only output and context growth are explicit',()=>{
|
||||
const d={...nodeData('llm','agent'),mode:'custom',customFirstMs:100,customTps:100,outputTokens:11,maxSteps:3,expectedSteps:2,toolMs:50,contextGrowth:20,inputTokens:100,finalOnly:true};
|
||||
const r=estimateNode(d,g);near(r.end,700);near(r.first,600);assert.deepEqual(r.steps.map(x=>x.input),[100,120,140]);
|
||||
const partial=estimateNode({...d,finalOnly:false},g);near(partial.first,100);near(partial.end,700);
|
||||
const s=simulateGraph([node('agent','llm',d),node('out','output',{outputType:'text',durationMs:0})],[edge('agent','out','first')],{...g,loopPlan:'expected'});
|
||||
near(s.first,350);near(s.maxFirst,600);near(s.maxComplete,700);near(s.results.agent.end,450);
|
||||
test("loop maximum, inter-step tools, final-only output and context growth are explicit", () => {
|
||||
const d = {
|
||||
...nodeData("llm", "agent"),
|
||||
mode: "custom",
|
||||
customFirstMs: 100,
|
||||
customTps: 100,
|
||||
outputTokens: 11,
|
||||
maxSteps: 3,
|
||||
expectedSteps: 2,
|
||||
toolMs: 50,
|
||||
contextGrowth: 20,
|
||||
inputTokens: 100,
|
||||
finalOnly: true,
|
||||
};
|
||||
const r = estimateNode(d, g);
|
||||
near(r.end, 700);
|
||||
near(r.first, 600);
|
||||
assert.deepEqual(
|
||||
r.steps.map((x) => x.input),
|
||||
[100, 120, 140],
|
||||
);
|
||||
const partial = estimateNode({ ...d, finalOnly: false }, g);
|
||||
near(partial.first, 100);
|
||||
near(partial.end, 700);
|
||||
const s = simulateGraph(
|
||||
[
|
||||
node("agent", "llm", d),
|
||||
node("out", "output", { outputType: "text", durationMs: 0 }),
|
||||
],
|
||||
[edge("agent", "out", "first")],
|
||||
{ ...g, loopPlan: "expected" },
|
||||
);
|
||||
near(s.first, 350);
|
||||
near(s.maxFirst, 600);
|
||||
near(s.maxComplete, 700);
|
||||
near(s.results.agent.end, 450);
|
||||
});
|
||||
test('hidden reasoning delays the first visible token',()=>{
|
||||
const d={...nodeData('llm','reasoner'),mode:'custom',customFirstMs:100,customTps:100,outputTokens:11,reasoningTokens:50};
|
||||
const r=estimateNode(d,g);near(r.first,600);near(r.end,700);
|
||||
test("hidden reasoning delays the first visible token", () => {
|
||||
const d = {
|
||||
...nodeData("llm", "reasoner"),
|
||||
mode: "custom",
|
||||
customFirstMs: 100,
|
||||
customTps: 100,
|
||||
outputTokens: 11,
|
||||
reasoningTokens: 50,
|
||||
};
|
||||
const r = estimateNode(d, g);
|
||||
near(r.first, 600);
|
||||
near(r.end, 700);
|
||||
});
|
||||
test('shared model resources serialize parallel requests and change critical dependency',()=>{
|
||||
const n=[node('in','input'),node('a','llm',{mode:'custom',customFirstMs:100,customTps:100,outputTokens:41}),node('b','llm',{mode:'custom',customFirstMs:100,customTps:100,outputTokens:41}),node('out','output',{outputType:'text',durationMs:0})];
|
||||
const e=[edge('in','a'),edge('in','b'),edge('a','out'),edge('b','out')];
|
||||
near(simulateGraph(n,e,g).complete,500);
|
||||
const s=simulateGraph(n,e,{...g,sharedGPU:true});near(s.complete,1000);near(s.results.b.resourceWait,500);
|
||||
test("shared model resources serialize parallel requests and change critical dependency", () => {
|
||||
const n = [
|
||||
node("in", "input"),
|
||||
node("a", "llm", {
|
||||
mode: "custom",
|
||||
customFirstMs: 100,
|
||||
customTps: 100,
|
||||
outputTokens: 41,
|
||||
}),
|
||||
node("b", "llm", {
|
||||
mode: "custom",
|
||||
customFirstMs: 100,
|
||||
customTps: 100,
|
||||
outputTokens: 41,
|
||||
}),
|
||||
node("out", "output", { outputType: "text", durationMs: 0 }),
|
||||
];
|
||||
const e = [
|
||||
edge("in", "a"),
|
||||
edge("in", "b"),
|
||||
edge("a", "out"),
|
||||
edge("b", "out"),
|
||||
];
|
||||
near(simulateGraph(n, e, g).complete, 500);
|
||||
const s = simulateGraph(n, e, { ...g, sharedGPU: true });
|
||||
near(s.complete, 1000);
|
||||
near(s.results.b.resourceWait, 500);
|
||||
});
|
||||
test('source text TTFT is not interpreted as first playable audio',()=>{
|
||||
const r=estimateNode({...nodeData('speech','LFM3'),mode:'reference',profileId:'lfm3',size:2},g);
|
||||
assert.notEqual(r.first,50);assert.ok(r.assumptions.some(x=>x.includes('只给首token')));
|
||||
test("source text TTFT is not interpreted as first playable audio", () => {
|
||||
const r = estimateNode(
|
||||
{
|
||||
...nodeData("speech", "LFM3"),
|
||||
mode: "reference",
|
||||
profileId: "lfm3",
|
||||
size: 2,
|
||||
},
|
||||
g,
|
||||
);
|
||||
assert.notEqual(r.first, 50);
|
||||
assert.ok(r.assumptions.some((x) => x.includes("只给首token")));
|
||||
});
|
||||
test('RTF-only reference uses its real factor without claiming first audio',()=>{
|
||||
const r=estimateNode({...nodeData('speech','MiniCPM'),mode:'reference',profileId:'minicpm',size:9},g);
|
||||
near(r.rtf,.21);assert.ok(r.assumptions.some(x=>x.includes('来源只给RTF')));assert.notEqual(r.first,210);
|
||||
test("RTF-only reference uses its real factor without claiming first audio", () => {
|
||||
const r = estimateNode(
|
||||
{
|
||||
...nodeData("speech", "MiniCPM"),
|
||||
mode: "reference",
|
||||
profileId: "minicpm",
|
||||
size: 9,
|
||||
},
|
||||
g,
|
||||
);
|
||||
near(r.rtf, 0.21);
|
||||
assert.ok(r.assumptions.some((x) => x.includes("来源只给RTF")));
|
||||
assert.notEqual(r.first, 210);
|
||||
});
|
||||
test('program scaling is global, and disconnected work is called out',()=>{
|
||||
const n=[node('a','program',{durationMs:100}),node('orphan','program',{durationMs:20}),node('out','output',{durationMs:0,outputType:'text'})];
|
||||
const s=simulateGraph(n,[edge('a','out')],{...g,programScale:2});near(s.first,200);assert.ok(s.warnings.some(x=>x.includes('orphan未连到主输出')));
|
||||
test("program scaling is global, and disconnected work is called out", () => {
|
||||
const n = [
|
||||
node("a", "program", { durationMs: 100 }),
|
||||
node("orphan", "program", { durationMs: 20 }),
|
||||
node("out", "output", { durationMs: 0, outputType: "text" }),
|
||||
];
|
||||
const s = simulateGraph(n, [edge("a", "out")], { ...g, programScale: 2 });
|
||||
near(s.first, 200);
|
||||
assert.ok(s.warnings.some((x) => x.includes("orphan未连到主输出")));
|
||||
});
|
||||
test('audio component size and hardware settings affect engineering estimate',()=>{
|
||||
const a=estimateNode({...nodeData('asr','ASR'),hardware:'h20',size:.6},g);
|
||||
const b=estimateNode({...nodeData('asr','ASR'),hardware:'h20',size:2},g);
|
||||
assert.ok(b.end>a.end);const t1=estimateNode({...nodeData('tts','TTS'),cards:1},g),t2=estimateNode({...nodeData('tts','TTS'),cards:2},g);assert.ok(t2.end<t1.end);
|
||||
test("audio component size and hardware settings affect engineering estimate", () => {
|
||||
const a = estimateNode(
|
||||
{ ...nodeData("asr", "ASR"), hardware: "h20", size: 0.6 },
|
||||
g,
|
||||
);
|
||||
const b = estimateNode(
|
||||
{ ...nodeData("asr", "ASR"), hardware: "h20", size: 2 },
|
||||
g,
|
||||
);
|
||||
assert.ok(b.end > a.end);
|
||||
const t1 = estimateNode({ ...nodeData("tts", "TTS"), cards: 1 }, g),
|
||||
t2 = estimateNode({ ...nodeData("tts", "TTS"), cards: 2 }, g);
|
||||
assert.ok(t2.end < t1.end);
|
||||
});
|
||||
test('network and endpoint wait are counted once at their boundaries',()=>{
|
||||
const n=[node('in','input'),node('out','output',{durationMs:60,outputType:'text'})];
|
||||
near(simulateGraph(n,[edge('in','out','first')],{...g,vadMs:120,networkMs:20}).first,200);
|
||||
test("network and endpoint wait are counted once at their boundaries", () => {
|
||||
const n = [
|
||||
node("in", "input"),
|
||||
node("out", "output", { durationMs: 60, outputType: "text" }),
|
||||
];
|
||||
near(
|
||||
simulateGraph(n, [edge("in", "out", "first")], {
|
||||
...g,
|
||||
vadMs: 120,
|
||||
networkMs: 20,
|
||||
}).first,
|
||||
200,
|
||||
);
|
||||
});
|
||||
test('invalid cycles and dangling edges are rejected including imports',()=>{
|
||||
const n=[node('a','program'),node('b','program')];
|
||||
assert.ok(validateGraph(n,[edge('a','b'),edge('b','a')]).some(x=>x.includes('环')));
|
||||
assert.ok(validateGraph(n,[edge('a','missing')]).length);
|
||||
assert.ok(validateDocument({version:1,nodes:n,edges:[],globals:{computeScale:-1}}).length);
|
||||
test("invalid cycles and dangling edges are rejected including imports", () => {
|
||||
const n = [node("a", "program"), node("b", "program")];
|
||||
assert.ok(
|
||||
validateGraph(n, [edge("a", "b"), edge("b", "a")]).some((x) =>
|
||||
x.includes("环"),
|
||||
),
|
||||
);
|
||||
assert.ok(validateGraph(n, [edge("a", "missing")]).length);
|
||||
assert.ok(
|
||||
validateDocument({
|
||||
version: 1,
|
||||
nodes: n,
|
||||
edges: [],
|
||||
globals: { computeScale: -1 },
|
||||
}).length,
|
||||
);
|
||||
});
|
||||
test('memory over-capacity is surfaced rather than silently claiming deployability',()=>{
|
||||
const r=estimateNode({...nodeData('llm','405B'),size:405,hardware:'4090',cards:1},g);
|
||||
assert.ok(r.warnings.some(x=>x.includes('显存')));
|
||||
test("memory over-capacity is surfaced rather than silently claiming deployability", () => {
|
||||
const r = estimateNode(
|
||||
{ ...nodeData("llm", "405B"), size: 405, hardware: "4090", cards: 1 },
|
||||
g,
|
||||
);
|
||||
assert.ok(r.warnings.some((x) => x.includes("显存")));
|
||||
});
|
||||
test('source records have provenance and do not confuse the units of metrics',()=>{
|
||||
const ids=new Set();
|
||||
for(const r of RECORDS){assert.ok(!ids.has(r.id));ids.add(r.id);assert.ok(SOURCES[r.source]?.url.startsWith('https://'));assert.ok(r.scope);if(r.evidence==='unknown')assert.equal(r.value,null);else assert.ok(r.value>0);}
|
||||
const q=RECORDS.find(r=>r.id==='q3-8');near(q.value,150.25);assert.equal(q.metric,'tps');
|
||||
test("source records have provenance and do not confuse the units of metrics", () => {
|
||||
const ids = new Set();
|
||||
for (const r of RECORDS) {
|
||||
assert.ok(!ids.has(r.id));
|
||||
ids.add(r.id);
|
||||
assert.ok(SOURCES[r.source]?.url.startsWith("https://"));
|
||||
assert.ok(r.scope);
|
||||
if (r.evidence === "unknown") assert.equal(r.value, null);
|
||||
else assert.ok(r.value > 0);
|
||||
}
|
||||
const q = RECORDS.find((r) => r.id === "q3-8");
|
||||
near(q.value, 150.25);
|
||||
assert.equal(q.metric, "tps");
|
||||
});
|
||||
test('all presets compute finite targets and survive JSON round trip',()=>{
|
||||
for(const key of Object.keys(PRESETS)){const d=makeDocument(key);assert.deepEqual(validateDocument(d),[]);const s=simulateGraph(d.nodes,d.edges,d.globals);assert.deepEqual(s.errors,[]);assert.ok(Number.isFinite(s.first));assert.ok(Number.isFinite(s.complete));assert.deepEqual(validateDocument(JSON.parse(JSON.stringify(d))),[]);assert.equal(autoLayout(d.nodes,d.edges).length,d.nodes.length);}
|
||||
test("all presets compute finite targets and survive JSON round trip", () => {
|
||||
for (const key of Object.keys(PRESETS)) {
|
||||
const d = makeDocument(key);
|
||||
assert.deepEqual(validateDocument(d), []);
|
||||
const s = simulateGraph(d.nodes, d.edges, d.globals);
|
||||
assert.deepEqual(s.errors, []);
|
||||
assert.ok(Number.isFinite(s.first));
|
||||
assert.ok(Number.isFinite(s.complete));
|
||||
assert.deepEqual(validateDocument(JSON.parse(JSON.stringify(d))), []);
|
||||
assert.equal(autoLayout(d.nodes, d.edges).length, d.nodes.length);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user