wuyang
367 lines
20 KiB
JavaScript
367 lines
20 KiB
JavaScript
import { writeFileSync } from "node:fs";
|
||
|
||
const cdpPort = process.env.CDP_PORT ?? "9227";
|
||
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
|
||
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
|
||
const page = pages.find((entry) => entry.type === "page");
|
||
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
|
||
|
||
const socket = new WebSocket(page.webSocketDebuggerUrl);
|
||
await new Promise((resolve, reject) => {
|
||
socket.addEventListener("open", resolve, { once: true });
|
||
socket.addEventListener("error", reject, { once: true });
|
||
});
|
||
|
||
let nextId = 0;
|
||
const pending = new Map();
|
||
const exceptions = [];
|
||
socket.addEventListener("message", (event) => {
|
||
const message = JSON.parse(event.data);
|
||
if (message.id && pending.has(message.id)) {
|
||
const { resolve, reject } = pending.get(message.id);
|
||
pending.delete(message.id);
|
||
if (message.error) reject(new Error(message.error.message));
|
||
else resolve(message.result);
|
||
}
|
||
if (message.method === "Runtime.exceptionThrown") {
|
||
exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
|
||
}
|
||
});
|
||
|
||
const command = (method, params = {}) => new Promise((resolve, reject) => {
|
||
const id = ++nextId;
|
||
pending.set(id, { resolve, reject });
|
||
socket.send(JSON.stringify({ id, method, params }));
|
||
});
|
||
const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||
const evaluate = async (expression) => {
|
||
const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
|
||
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
|
||
return result.result.value;
|
||
};
|
||
const navigate = async (path) => {
|
||
await command("Page.navigate", { url: `${baseUrl}${path}` });
|
||
for (let attempt = 0; attempt < 80; attempt += 1) {
|
||
await pause(100);
|
||
if (await evaluate("document.readyState === 'complete'")) return;
|
||
}
|
||
throw new Error(`${path} 加载超时`);
|
||
};
|
||
const screenshot = async (path, beyond = false) => {
|
||
const result = await command("Page.captureScreenshot", { format: "png", captureBeyondViewport: beyond });
|
||
writeFileSync(path, Buffer.from(result.data, "base64"));
|
||
};
|
||
|
||
await command("Page.enable");
|
||
await command("Runtime.enable");
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 1440,
|
||
height: 1100,
|
||
deviceScaleFactor: 1,
|
||
mobile: false,
|
||
});
|
||
|
||
await navigate("/k3/");
|
||
await screenshot("/tmp/llm-atlas-k3-desktop.png");
|
||
|
||
const overview = await evaluate(`(() => ({
|
||
title: document.querySelector("h1")?.textContent.trim(),
|
||
sections: document.querySelectorAll(".article-section").length,
|
||
tocLinks: document.querySelectorAll(".side-rail a").length,
|
||
ledgers: document.querySelectorAll(".ledger-card").length,
|
||
reportMap: document.querySelectorAll(".report-map > article").length,
|
||
figureAtlas: document.querySelectorAll(".figure-atlas > article").length,
|
||
paperLinks: document.querySelectorAll("#papers .paper-row").length,
|
||
paperGroups: document.querySelectorAll("#papers .paper-group").length,
|
||
labTabs: document.querySelectorAll("[data-k3-tab]").length,
|
||
labPanels: document.querySelectorAll("[data-k3-panel]").length,
|
||
artifactTabs: document.querySelectorAll("[data-artifact-tab]").length,
|
||
artifactPanels: document.querySelectorAll("[data-artifact-panel]").length,
|
||
artifactLayers: document.querySelectorAll("[data-layer-cell]").length,
|
||
artifactMismatch: document.querySelector("#artifacts")?.textContent.includes("A_log [128] ≠ expected [96]"),
|
||
attnresTabs: document.querySelectorAll("[data-attnres-tab]").length,
|
||
attnresPanels: document.querySelectorAll("[data-attnres-panel]").length,
|
||
nativeVisionCorrected: document.body.textContent.includes("MoonViT‑V2 从头训练") &&
|
||
document.body.textContent.includes("同一个 next-token prediction objective"),
|
||
staleVisionClaim: document.body.textContent.includes("先固定语言模型训练视觉组件"),
|
||
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||
}))()`);
|
||
|
||
const labs = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-k3-lab]");
|
||
const panel = () => root.querySelector("[data-k3-panel]:not([hidden])").dataset.k3Panel;
|
||
const text = (selector) => root.querySelector(selector).textContent.trim();
|
||
const input = (selector, value) => {
|
||
const node = root.querySelector(selector);
|
||
node.value = value;
|
||
node.dispatchEvent(new Event("input", { bubbles: true }));
|
||
};
|
||
|
||
const memoryInitial = {
|
||
panel: panel(),
|
||
additive: text("[data-additive-result]"),
|
||
delta: text("[data-delta-result]"),
|
||
additiveError: text("[data-additive-error]"),
|
||
deltaError: text("[data-delta-error]"),
|
||
};
|
||
input("[data-memory-writes]", "12");
|
||
const memoryChanged = { additive: text("[data-additive-result]"), delta: text("[data-delta-result]") };
|
||
|
||
root.querySelector('[data-k3-tab="decay"]').click();
|
||
const decayInitial = { panel: panel(), verdict: text("[data-decay-verdict] b"), log: text("[data-decay-log]"), recip: text("[data-decay-recip]") };
|
||
input("[data-decay-g]", "-120");
|
||
input("[data-decay-tile]", "32");
|
||
const decayRisk = { verdict: text("[data-decay-verdict] b"), log: text("[data-decay-log]") };
|
||
|
||
root.querySelector('[data-k3-tab="depth"]').click();
|
||
const depthInitial = { sources: text("[data-block-sources]"), contract: text("[data-depth-reduction]") };
|
||
input("[data-depth-block]", "6");
|
||
const depthChanged = { sources: text("[data-block-sources]"), contract: text("[data-depth-reduction]") };
|
||
|
||
root.querySelector('[data-k3-tab="width"]').click();
|
||
const widthInitial = { conventional: text("[data-width-conventional]"), latent: text("[data-width-latent]"), reduction: text("[data-width-reduction]") };
|
||
input("[data-width-latent]", "7168");
|
||
const widthFull = { reduction: text("[data-width-reduction]") };
|
||
|
||
root.querySelector('[data-k3-tab="situ"]').click();
|
||
const situInitial = { bound: text("[data-situ-bound]"), value: text("[data-situ-value]"), swiglu: text("[data-situ-swiglu]") };
|
||
input("[data-situ-x]", "1000");
|
||
const situLarge = { bound: text("[data-situ-bound]"), value: text("[data-situ-value]"), swiglu: text("[data-situ-swiglu]") };
|
||
|
||
root.querySelector('[data-k3-tab="qb"]').click();
|
||
const qbInitial = { before: text("[data-qb-before-copy]"), after: text("[data-qb-after-copy]"), bias: text("[data-qb-bias]"), gap: text("[data-qb-gap]") };
|
||
input("[data-qb-strength]", "0");
|
||
const qbOff = { after: text("[data-qb-after-copy]"), gap: text("[data-qb-gap]") };
|
||
|
||
root.querySelector('[data-k3-tab="rl"]').click();
|
||
const rlInitial = {
|
||
complete: text("[data-rl-complete]"),
|
||
paused: text("[data-rl-paused]"),
|
||
budget: text("[data-rl-budget-copy]"),
|
||
reward: text("[data-rl-reward]"),
|
||
};
|
||
input("[data-rl-tokens]", "40");
|
||
const rlOver = { budget: text("[data-rl-budget-copy]") };
|
||
|
||
root.querySelector('[data-k3-tab="cache"]').click();
|
||
const cacheInitial = {
|
||
mla: text("[data-cache-mla]"),
|
||
kda: text("[data-cache-kda]"),
|
||
hit: text("[data-cache-hit]"),
|
||
recompute: text("[data-cache-recompute]"),
|
||
};
|
||
input("[data-cache-checkpoint]", "3");
|
||
const cacheSparse = { kda: text("[data-cache-kda]"), hit: text("[data-cache-hit]"), recompute: text("[data-cache-recompute]") };
|
||
|
||
const first = root.querySelector('[data-k3-tab="memory"]');
|
||
first.focus();
|
||
first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
||
return {
|
||
memoryInitial, memoryChanged, decayInitial, decayRisk, depthInitial, depthChanged,
|
||
widthInitial, widthFull, situInitial, situLarge, qbInitial, qbOff, rlInitial, rlOver,
|
||
cacheInitial, cacheSparse,
|
||
keyboardSelected: root.querySelector('[data-k3-tab][aria-selected="true"]').dataset.k3Tab,
|
||
keyboardVisible: panel(),
|
||
};
|
||
})()`);
|
||
|
||
const artifacts = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-k3-artifact-lab]");
|
||
const panel = () => root.querySelector("[data-artifact-panel]:not([hidden])").dataset.artifactPanel;
|
||
const text = (selector) => root.querySelector(selector).textContent.trim();
|
||
const input = (selector, value) => {
|
||
const node = root.querySelector(selector);
|
||
node.value = value;
|
||
node.dispatchEvent(new Event("input", { bubbles: true }));
|
||
};
|
||
|
||
const initial = {
|
||
panel: panel(),
|
||
layer: text("[data-layer-number]"),
|
||
attention: text("[data-layer-attention]"),
|
||
ffn: text("[data-layer-ffn]"),
|
||
block: text("[data-layer-block]"),
|
||
};
|
||
input("[data-layer-slider]", "93");
|
||
const terminal = {
|
||
layer: text("[data-layer-number]"),
|
||
attention: text("[data-layer-attention]"),
|
||
ffn: text("[data-layer-ffn]"),
|
||
copy: text("[data-layer-special]"),
|
||
};
|
||
|
||
root.querySelector('[data-artifact-tab="tensors"]').click();
|
||
const tensors = {
|
||
panel: panel(),
|
||
groups: root.querySelectorAll("[data-tensor-tab]").length,
|
||
expertRows: root.querySelector('[data-tensor-panel="expert"]').querySelectorAll(":scope > div").length,
|
||
entries: root.textContent.includes("497,220"),
|
||
share: root.textContent.includes("92.67%"),
|
||
};
|
||
root.querySelector('[data-tensor-tab="mla"]').click();
|
||
const mla = {
|
||
visible: !root.querySelector('[data-tensor-panel="mla"]').hidden,
|
||
rows: root.querySelector('[data-tensor-panel="mla"]').querySelectorAll(":scope > div").length,
|
||
has576: root.querySelector('[data-tensor-panel="mla"]').textContent.includes("576 × 7168"),
|
||
};
|
||
|
||
root.querySelector('[data-artifact-tab="parameters"]').click();
|
||
const parameterInitial = {
|
||
panel: panel(),
|
||
shape: text("[data-parameter-shape]"),
|
||
conflict: root.textContent.includes("A_log [128]") && root.textContent.includes("A_log [H] = [96]"),
|
||
};
|
||
input("[data-parameter-select]", "dt");
|
||
const parameterChanged = {
|
||
shape: text("[data-parameter-shape]"),
|
||
count: text("[data-parameter-count]"),
|
||
};
|
||
|
||
root.querySelector('[data-artifact-tab="reproduction"]').click();
|
||
const reproductionInitial = {
|
||
panel: panel(),
|
||
flash: text("[data-benchmark-flash]"),
|
||
fla: text("[data-benchmark-fla]"),
|
||
speedup: text("[data-benchmark-speedup]"),
|
||
localMean: text("[data-local-mean]"),
|
||
localP95: text("[data-local-p95]"),
|
||
localThroughput: text("[data-local-throughput]"),
|
||
exactSuite: root.textContent.includes("6 / 6 PASS") && root.textContent.includes("MAX ABS ERROR"),
|
||
cv: text("[data-router-cv]"),
|
||
zero: text("[data-router-zero]"),
|
||
};
|
||
input("[data-benchmark-device]", "gb200");
|
||
input("[data-benchmark-case]", "Varlen, \\\`seq_lens\\\`=\\\`1024 x 8\\\`");
|
||
input("[data-local-case]", "k3_varlen_shape");
|
||
input("[data-local-state]", "fp32_state");
|
||
input("[data-router-mode]", "bias");
|
||
const reproductionChanged = {
|
||
flash: text("[data-benchmark-flash]"),
|
||
speedup: text("[data-benchmark-speedup]"),
|
||
localMean: text("[data-local-mean]"),
|
||
localP95: text("[data-local-p95]"),
|
||
localMode: text("[data-local-mode-copy]"),
|
||
cv: text("[data-router-cv]"),
|
||
zero: text("[data-router-zero]"),
|
||
};
|
||
|
||
const first = root.querySelector('[data-artifact-tab="layers"]');
|
||
first.focus();
|
||
first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
||
return {
|
||
initial, terminal, tensors, mla, parameterInitial, parameterChanged,
|
||
reproductionInitial, reproductionChanged,
|
||
keyboardSelected: root.querySelector('[data-artifact-tab][aria-selected="true"]').dataset.artifactTab,
|
||
keyboardVisible: panel(),
|
||
};
|
||
})()`);
|
||
|
||
await evaluate(`(() => {
|
||
document.querySelector("[data-k3-lab]").scrollIntoView({ block: "start", behavior: "instant" });
|
||
window.scrollBy(0, -82);
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-k3-lab-desktop.png");
|
||
await evaluate(`(() => {
|
||
document.querySelector("[data-k3-artifact-lab]").scrollIntoView({ block: "start", behavior: "instant" });
|
||
document.querySelector('[data-artifact-tab="reproduction"]')?.click();
|
||
window.scrollBy(0, -82);
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-k3-artifact-desktop.png");
|
||
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 390,
|
||
height: 844,
|
||
deviceScaleFactor: 1,
|
||
mobile: true,
|
||
});
|
||
await navigate("/k3/");
|
||
const mobile = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-k3-lab]");
|
||
root.scrollIntoView({ block: "start", behavior: "instant" });
|
||
const toggle = document.querySelector("#menu-toggle");
|
||
toggle?.click();
|
||
return {
|
||
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||
menuVisible: getComputedStyle(toggle).display !== "none",
|
||
menuOpen: toggle.getAttribute("aria-expanded"),
|
||
tabs: root.querySelectorAll("[data-k3-tab]").length,
|
||
artifactTabs: document.querySelectorAll("[data-artifact-tab]").length,
|
||
artifactLayers: document.querySelectorAll("[data-layer-cell]").length,
|
||
attnresTabs: document.querySelectorAll("[data-attnres-tab]").length,
|
||
offenders: [...document.querySelectorAll("body *")]
|
||
.filter((node) => !node.closest(".paper-chain, .spec-table-wrap, .cache-strip, .architecture-explorer, [data-k3-lab], [data-k3-artifact-lab], [data-attnres-lab]"))
|
||
.filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
|
||
.slice(0, 15)
|
||
.map((node) => ({
|
||
tag: node.tagName,
|
||
className: typeof node.className === "string" ? node.className : "",
|
||
right: Math.round(node.getBoundingClientRect().right),
|
||
width: Math.round(node.getBoundingClientRect().width),
|
||
})),
|
||
};
|
||
})()`);
|
||
await evaluate(`(() => {
|
||
document.querySelector("#menu-toggle")?.click();
|
||
window.scrollBy(0, -82);
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-k3-mobile.png");
|
||
await evaluate(`(() => {
|
||
document.querySelector("[data-k3-artifact-lab]").scrollIntoView({ block: "start", behavior: "instant" });
|
||
document.querySelector('[data-artifact-tab="reproduction"]')?.click();
|
||
window.scrollBy(0, -64);
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-k3-artifact-mobile.png");
|
||
|
||
const report = { overview, labs, artifacts, mobile, exceptions };
|
||
console.log(JSON.stringify(report, null, 2));
|
||
|
||
const numeric = (text) => Number.parseFloat(text.replaceAll(",", "").replace("−", "-"));
|
||
const failures = [];
|
||
if (!overview.title.includes("因果环节")) failures.push("K3 二轮标题异常");
|
||
if (overview.sections !== 33 || overview.tocLinks !== 33) failures.push("32 个编号专题加阅读链的目录结构异常");
|
||
if (overview.ledgers !== 32 || overview.reportMap !== 9) failures.push("32 张问题账或报告地图异常");
|
||
if (overview.figureAtlas !== 21 || overview.paperLinks !== 100 || overview.paperGroups < 12) failures.push("图表审计或 100 节点阅读链异常");
|
||
if (overview.labTabs !== 8 || overview.labPanels !== 8) failures.push("八联实验结构异常");
|
||
if (overview.artifactTabs !== 4 || overview.artifactPanels !== 4 || overview.artifactLayers !== 93 || !overview.artifactMismatch) failures.push("开放工件四视图、93 层条带或形状冲突异常");
|
||
if (overview.attnresTabs !== 5 || overview.attnresPanels !== 5) failures.push("AttnRes 独立实验五视图异常");
|
||
if (!overview.nativeVisionCorrected || overview.staleVisionClaim) failures.push("原生多模态纠错未生效或旧错误残留");
|
||
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在文档级横向溢出");
|
||
if (labs.memoryInitial.panel !== "memory" || numeric(labs.memoryInitial.additiveError) <= numeric(labs.memoryInitial.deltaError)) failures.push("Delta memory 初始递推异常");
|
||
if (numeric(labs.memoryChanged.additive) <= numeric(labs.memoryInitial.additive) || numeric(labs.memoryChanged.delta) <= numeric(labs.memoryInitial.delta)) failures.push("Delta memory 控件未更新");
|
||
if (labs.decayInitial.panel !== "decay" || labs.decayInitial.verdict !== "WITHIN RANGE" || numeric(labs.decayInitial.log) !== -80) failures.push("K3 默认 bounded decay 范围异常");
|
||
if (labs.decayRisk.verdict !== "OVERFLOW RISK" || numeric(labs.decayRisk.log) !== -384) failures.push("bounded decay 风险探针异常");
|
||
if (numeric(labs.depthInitial.sources) !== 9 || numeric(labs.depthChanged.sources) !== 17) failures.push("Block AttnRes 来源计数异常");
|
||
if (numeric(labs.widthInitial.reduction) !== 50 || numeric(labs.widthFull.reduction) !== 0) failures.push("LatentMoE payload 账异常");
|
||
if (numeric(labs.situInitial.bound) !== 100 || numeric(labs.situLarge.value) > 100.01 || numeric(labs.situLarge.swiglu) <= 1000) failures.push("SiTU formal bound 异常");
|
||
if (!labs.qbInitial.before.includes("4, 3, 1, 0") || numeric(labs.qbInitial.gap) > 1 || numeric(labs.qbOff.gap) < 3) failures.push("Quantile Balancing toy route 异常");
|
||
if (!labs.rlInitial.complete.includes("192") || !labs.rlInitial.paused.includes("64") || numeric(labs.rlInitial.reward) !== 0.693) failures.push("MOPD / partial rollout 初始账异常");
|
||
if (!labs.rlOver.budget.includes("reward 改为 −1")) failures.push("Reasoning effort 超预算未触发");
|
||
if (numeric(labs.cacheInitial.hit) !== 2560 || !labs.cacheInitial.recompute.includes("256")) failures.push("Hybrid prefix cache 默认命中异常");
|
||
if (numeric(labs.cacheSparse.hit) >= numeric(labs.cacheInitial.hit) || numeric(labs.cacheSparse.recompute) <= numeric(labs.cacheInitial.recompute)) failures.push("稀疏 KDA checkpoint 未降低 joint hit");
|
||
if (labs.keyboardSelected !== "decay" || labs.keyboardVisible !== "decay") failures.push("实验键盘 tab 导航异常");
|
||
if (artifacts.initial.panel !== "layers" || numeric(artifacts.initial.layer) !== 1 || artifacts.initial.attention !== "KDA" || artifacts.initial.ffn !== "DENSE") failures.push("开放工件初始层视图异常");
|
||
if (numeric(artifacts.terminal.layer) !== 93 || artifacts.terminal.attention !== "MLA" || artifacts.terminal.ffn !== "MOE" || !artifacts.terminal.copy.includes("L92 / L93")) failures.push("K3 末层真实配置条带异常");
|
||
if (artifacts.tensors.panel !== "tensors" || artifacts.tensors.groups !== 3 || artifacts.tensors.expertRows !== 6 || !artifacts.tensors.entries || !artifacts.tensors.share) failures.push("checkpoint tensor anatomy 异常");
|
||
if (!artifacts.mla.visible || artifacts.mla.rows !== 5 || !artifacts.mla.has576) failures.push("MLA header shape 视图异常");
|
||
if (artifacts.parameterInitial.panel !== "parameters" || artifacts.parameterInitial.shape !== "[128] F32" || !artifacts.parameterInitial.conflict) failures.push("A_log 工件冲突审计异常");
|
||
if (artifacts.parameterChanged.shape !== "[96,128] F32" || !artifacts.parameterChanged.count.includes("12,288")) failures.push("真实 dt_bias 参数切换异常");
|
||
if (artifacts.reproductionInitial.panel !== "reproduction" || numeric(artifacts.reproductionInitial.speedup) !== 1.85 || numeric(artifacts.reproductionInitial.localMean) < 2.6 || !artifacts.reproductionInitial.exactSuite || numeric(artifacts.reproductionInitial.cv) < 2) failures.push("FlashKDA H20、本机 exact suite 或 router 初始探针异常");
|
||
if (numeric(artifacts.reproductionChanged.speedup) !== 3.27 || numeric(artifacts.reproductionChanged.flash) !== 0.7064 || numeric(artifacts.reproductionChanged.localMean) >= numeric(artifacts.reproductionInitial.localMean) || !artifacts.reproductionChanged.localMode.includes("FP32 state") || numeric(artifacts.reproductionChanged.cv) <= numeric(artifacts.reproductionInitial.cv) || numeric(artifacts.reproductionChanged.zero) <= numeric(artifacts.reproductionInitial.zero)) failures.push("GB200 benchmark、本机 varlen/state 或 synthetic router counterexample 未更新");
|
||
if (artifacts.keyboardSelected !== "tensors" || artifacts.keyboardVisible !== "tensors") failures.push("开放工件键盘 tab 导航异常");
|
||
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 8 || mobile.artifactTabs !== 4 || mobile.artifactLayers !== 93 || mobile.attnresTabs !== 5) failures.push("移动端导航或实验异常");
|
||
if (mobile.offenders.length) failures.push(`移动端越界元素：${JSON.stringify(mobile.offenders)}`);
|
||
if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);
|
||
|
||
if (failures.length) {
|
||
console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
|
||
process.exitCode = 1;
|
||
} else {
|
||
console.log("\nPASS K3 browser regression");
|
||
}
|
||
|
||
socket.close();
|