Files
llm-atlas/scripts/check-k3-browser.mjs

386 lines
22 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { writeFileSync } from "node:fs";
const cdpPort = process.env.CDP_PORT ?? "9227";
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
const page = pages.find((entry) => entry.type === "page");
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
const socket = new WebSocket(page.webSocketDebuggerUrl);
await new Promise((resolve, reject) => {
socket.addEventListener("open", resolve, { once: true });
socket.addEventListener("error", reject, { once: true });
});
let nextId = 0;
const pending = new Map();
const exceptions = [];
socket.addEventListener("message", (event) => {
const message = JSON.parse(event.data);
if (message.id && pending.has(message.id)) {
const { resolve, reject } = pending.get(message.id);
pending.delete(message.id);
if (message.error) reject(new Error(message.error.message));
else resolve(message.result);
}
if (message.method === "Runtime.exceptionThrown") {
exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
}
});
const command = (method, params = {}) => new Promise((resolve, reject) => {
const id = ++nextId;
pending.set(id, { resolve, reject });
socket.send(JSON.stringify({ id, method, params }));
});
const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
const evaluate = async (expression) => {
const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
return result.result.value;
};
const navigate = async (path) => {
await command("Page.navigate", { url: `${baseUrl}${path}` });
for (let attempt = 0; attempt < 80; attempt += 1) {
await pause(100);
if (await evaluate("document.readyState === 'complete'")) return;
}
throw new Error(`${path} 加载超时`);
};
const screenshot = async (path, beyond = false) => {
const result = await command("Page.captureScreenshot", { format: "png", captureBeyondViewport: beyond });
writeFileSync(path, Buffer.from(result.data, "base64"));
};
await command("Page.enable");
await command("Runtime.enable");
await command("Emulation.setDeviceMetricsOverride", {
width: 1440,
height: 1100,
deviceScaleFactor: 1,
mobile: false,
});
await navigate("/k3/");
await screenshot("/tmp/llm-atlas-k3-desktop.png");
const overview = await evaluate(`(() => ({
title: document.querySelector("h1")?.textContent.trim(),
sections: document.querySelectorAll(".article-section").length,
tocLinks: document.querySelectorAll(".side-rail a").length,
ledgers: document.querySelectorAll(".ledger-card").length,
reportMap: document.querySelectorAll(".report-map > article").length,
figureAtlas: document.querySelectorAll(".figure-atlas > article").length,
paperLinks: document.querySelectorAll("#papers .paper-row").length,
paperGroups: document.querySelectorAll("#papers .paper-group").length,
labTabs: document.querySelectorAll("[data-k3-tab]").length,
labPanels: document.querySelectorAll("[data-k3-panel]").length,
artifactTabs: document.querySelectorAll("[data-artifact-tab]").length,
artifactPanels: document.querySelectorAll("[data-artifact-panel]").length,
artifactLayers: document.querySelectorAll("[data-layer-cell]").length,
artifactMismatch: document.querySelector("#artifacts")?.textContent.includes("A_log [128] ≠ expected [96]"),
attnresTabs: document.querySelectorAll("[data-attnres-tab]").length,
attnresPanels: document.querySelectorAll("[data-attnres-panel]").length,
gradientTabs: document.querySelectorAll("[data-gradient-tab]").length,
gradientPanels: document.querySelectorAll("[data-gradient-panel]").length,
spikeTabs: document.querySelectorAll("[data-spike-tab]").length,
spikePanels: document.querySelectorAll("[data-spike-panel]").length,
localPathTabs: document.querySelectorAll("[data-local-tab]").length,
localPathPanels: document.querySelectorAll("[data-local-panel]").length,
localPathVerdict: document.querySelector("#attnres-local-path")?.textContent.includes("localization 未建立"),
forwardTabs: document.querySelectorAll("[data-forward-tab]").length,
forwardPanels: document.querySelectorAll("[data-forward-panel]").length,
forwardVerdict: document.querySelector("#attnres-forward")?.textContent.includes("6 / 6 PASS") &&
document.querySelector("#attnres-forward")?.textContent.includes("reduced protocol only"),
nativeVisionCorrected: document.body.textContent.includes("MoonViT‑V2 从头训练") &&
document.body.textContent.includes("同一个 next-token prediction objective"),
staleVisionClaim: document.body.textContent.includes("先固定语言模型训练视觉组件"),
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
}))()`);
const labs = await evaluate(`(() => {
const root = document.querySelector("[data-k3-lab]");
const panel = () => root.querySelector("[data-k3-panel]:not([hidden])").dataset.k3Panel;
const text = (selector) => root.querySelector(selector).textContent.trim();
const input = (selector, value) => {
const node = root.querySelector(selector);
node.value = value;
node.dispatchEvent(new Event("input", { bubbles: true }));
};
const memoryInitial = {
panel: panel(),
additive: text("[data-additive-result]"),
delta: text("[data-delta-result]"),
additiveError: text("[data-additive-error]"),
deltaError: text("[data-delta-error]"),
};
input("[data-memory-writes]", "12");
const memoryChanged = { additive: text("[data-additive-result]"), delta: text("[data-delta-result]") };
root.querySelector('[data-k3-tab="decay"]').click();
const decayInitial = { panel: panel(), verdict: text("[data-decay-verdict] b"), log: text("[data-decay-log]"), recip: text("[data-decay-recip]") };
input("[data-decay-g]", "-120");
input("[data-decay-tile]", "32");
const decayRisk = { verdict: text("[data-decay-verdict] b"), log: text("[data-decay-log]") };
root.querySelector('[data-k3-tab="depth"]').click();
const depthInitial = { sources: text("[data-block-sources]"), contract: text("[data-depth-reduction]") };
input("[data-depth-block]", "6");
const depthChanged = { sources: text("[data-block-sources]"), contract: text("[data-depth-reduction]") };
root.querySelector('[data-k3-tab="width"]').click();
const widthInitial = { conventional: text("[data-width-conventional]"), latent: text("[data-width-latent]"), reduction: text("[data-width-reduction]") };
input("[data-width-latent]", "7168");
const widthFull = { reduction: text("[data-width-reduction]") };
root.querySelector('[data-k3-tab="situ"]').click();
const situInitial = { bound: text("[data-situ-bound]"), value: text("[data-situ-value]"), swiglu: text("[data-situ-swiglu]") };
input("[data-situ-x]", "1000");
const situLarge = { bound: text("[data-situ-bound]"), value: text("[data-situ-value]"), swiglu: text("[data-situ-swiglu]") };
root.querySelector('[data-k3-tab="qb"]').click();
const qbInitial = { before: text("[data-qb-before-copy]"), after: text("[data-qb-after-copy]"), bias: text("[data-qb-bias]"), gap: text("[data-qb-gap]") };
input("[data-qb-strength]", "0");
const qbOff = { after: text("[data-qb-after-copy]"), gap: text("[data-qb-gap]") };
root.querySelector('[data-k3-tab="rl"]').click();
const rlInitial = {
complete: text("[data-rl-complete]"),
paused: text("[data-rl-paused]"),
budget: text("[data-rl-budget-copy]"),
reward: text("[data-rl-reward]"),
};
input("[data-rl-tokens]", "40");
const rlOver = { budget: text("[data-rl-budget-copy]") };
root.querySelector('[data-k3-tab="cache"]').click();
const cacheInitial = {
mla: text("[data-cache-mla]"),
kda: text("[data-cache-kda]"),
hit: text("[data-cache-hit]"),
recompute: text("[data-cache-recompute]"),
};
input("[data-cache-checkpoint]", "3");
const cacheSparse = { kda: text("[data-cache-kda]"), hit: text("[data-cache-hit]"), recompute: text("[data-cache-recompute]") };
const first = root.querySelector('[data-k3-tab="memory"]');
first.focus();
first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
return {
memoryInitial, memoryChanged, decayInitial, decayRisk, depthInitial, depthChanged,
widthInitial, widthFull, situInitial, situLarge, qbInitial, qbOff, rlInitial, rlOver,
cacheInitial, cacheSparse,
keyboardSelected: root.querySelector('[data-k3-tab][aria-selected="true"]').dataset.k3Tab,
keyboardVisible: panel(),
};
})()`);
const artifacts = await evaluate(`(() => {
const root = document.querySelector("[data-k3-artifact-lab]");
const panel = () => root.querySelector("[data-artifact-panel]:not([hidden])").dataset.artifactPanel;
const text = (selector) => root.querySelector(selector).textContent.trim();
const input = (selector, value) => {
const node = root.querySelector(selector);
node.value = value;
node.dispatchEvent(new Event("input", { bubbles: true }));
};
const initial = {
panel: panel(),
layer: text("[data-layer-number]"),
attention: text("[data-layer-attention]"),
ffn: text("[data-layer-ffn]"),
block: text("[data-layer-block]"),
};
input("[data-layer-slider]", "93");
const terminal = {
layer: text("[data-layer-number]"),
attention: text("[data-layer-attention]"),
ffn: text("[data-layer-ffn]"),
copy: text("[data-layer-special]"),
};
root.querySelector('[data-artifact-tab="tensors"]').click();
const tensors = {
panel: panel(),
groups: root.querySelectorAll("[data-tensor-tab]").length,
expertRows: root.querySelector('[data-tensor-panel="expert"]').querySelectorAll(":scope > div").length,
entries: root.textContent.includes("497,220"),
share: root.textContent.includes("92.67%"),
};
root.querySelector('[data-tensor-tab="mla"]').click();
const mla = {
visible: !root.querySelector('[data-tensor-panel="mla"]').hidden,
rows: root.querySelector('[data-tensor-panel="mla"]').querySelectorAll(":scope > div").length,
has576: root.querySelector('[data-tensor-panel="mla"]').textContent.includes("576 × 7168"),
};
root.querySelector('[data-artifact-tab="parameters"]').click();
const parameterInitial = {
panel: panel(),
shape: text("[data-parameter-shape]"),
conflict: root.textContent.includes("A_log [128]") && root.textContent.includes("A_log [H] = [96]"),
};
input("[data-parameter-select]", "dt");
const parameterChanged = {
shape: text("[data-parameter-shape]"),
count: text("[data-parameter-count]"),
};
root.querySelector('[data-artifact-tab="reproduction"]').click();
const reproductionInitial = {
panel: panel(),
flash: text("[data-benchmark-flash]"),
fla: text("[data-benchmark-fla]"),
speedup: text("[data-benchmark-speedup]"),
localMean: text("[data-local-mean]"),
localP95: text("[data-local-p95]"),
localThroughput: text("[data-local-throughput]"),
exactSuite: root.textContent.includes("6 / 6 PASS") && root.textContent.includes("MAX ABS ERROR"),
cv: text("[data-router-cv]"),
zero: text("[data-router-zero]"),
};
input("[data-benchmark-device]", "gb200");
input("[data-benchmark-case]", "Varlen, \\\`seq_lens\\\`=\\\`1024 x 8\\\`");
input("[data-local-case]", "k3_varlen_shape");
input("[data-local-state]", "fp32_state");
input("[data-router-mode]", "bias");
const reproductionChanged = {
flash: text("[data-benchmark-flash]"),
speedup: text("[data-benchmark-speedup]"),
localMean: text("[data-local-mean]"),
localP95: text("[data-local-p95]"),
localMode: text("[data-local-mode-copy]"),
cv: text("[data-router-cv]"),
zero: text("[data-router-zero]"),
};
const first = root.querySelector('[data-artifact-tab="layers"]');
first.focus();
first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
return {
initial, terminal, tensors, mla, parameterInitial, parameterChanged,
reproductionInitial, reproductionChanged,
keyboardSelected: root.querySelector('[data-artifact-tab][aria-selected="true"]').dataset.artifactTab,
keyboardVisible: panel(),
};
})()`);
await evaluate(`(() => {
document.querySelector("[data-k3-lab]").scrollIntoView({ block: "start", behavior: "instant" });
window.scrollBy(0, -82);
})()`);
await pause(180);
await screenshot("/tmp/llm-atlas-k3-lab-desktop.png");
await evaluate(`(() => {
document.querySelector("[data-k3-artifact-lab]").scrollIntoView({ block: "start", behavior: "instant" });
document.querySelector('[data-artifact-tab="reproduction"]')?.click();
window.scrollBy(0, -82);
})()`);
await pause(180);
await screenshot("/tmp/llm-atlas-k3-artifact-desktop.png");
await command("Emulation.setDeviceMetricsOverride", {
width: 390,
height: 844,
deviceScaleFactor: 1,
mobile: true,
});
await navigate("/k3/");
const mobile = await evaluate(`(() => {
const root = document.querySelector("[data-k3-lab]");
root.scrollIntoView({ block: "start", behavior: "instant" });
const toggle = document.querySelector("#menu-toggle");
toggle?.click();
return {
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
menuVisible: getComputedStyle(toggle).display !== "none",
menuOpen: toggle.getAttribute("aria-expanded"),
tabs: root.querySelectorAll("[data-k3-tab]").length,
artifactTabs: document.querySelectorAll("[data-artifact-tab]").length,
artifactLayers: document.querySelectorAll("[data-layer-cell]").length,
attnresTabs: document.querySelectorAll("[data-attnres-tab]").length,
gradientTabs: document.querySelectorAll("[data-gradient-tab]").length,
spikeTabs: document.querySelectorAll("[data-spike-tab]").length,
localPathTabs: document.querySelectorAll("[data-local-tab]").length,
forwardTabs: document.querySelectorAll("[data-forward-tab]").length,
offenders: [...document.querySelectorAll("body *")]
.filter((node) => !node.closest(".paper-chain, .spec-table-wrap, .cache-strip, .architecture-explorer, [data-k3-lab], [data-k3-artifact-lab], [data-attnres-lab], [data-gradient-lab], [data-spike-lab], [data-local-path-lab], [data-forward-lab]"))
.filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
.slice(0, 15)
.map((node) => ({
tag: node.tagName,
className: typeof node.className === "string" ? node.className : "",
right: Math.round(node.getBoundingClientRect().right),
width: Math.round(node.getBoundingClientRect().width),
})),
};
})()`);
await evaluate(`(() => {
document.querySelector("#menu-toggle")?.click();
window.scrollBy(0, -82);
})()`);
await pause(180);
await screenshot("/tmp/llm-atlas-k3-mobile.png");
await evaluate(`(() => {
document.querySelector("[data-k3-artifact-lab]").scrollIntoView({ block: "start", behavior: "instant" });
document.querySelector('[data-artifact-tab="reproduction"]')?.click();
window.scrollBy(0, -64);
})()`);
await pause(180);
await screenshot("/tmp/llm-atlas-k3-artifact-mobile.png");
const report = { overview, labs, artifacts, mobile, exceptions };
console.log(JSON.stringify(report, null, 2));
const numeric = (text) => Number.parseFloat(text.replaceAll(",", "").replace("−", "-"));
const failures = [];
if (!overview.title.includes("因果环节")) failures.push("K3 二轮标题异常");
if (overview.sections !== 37 || overview.tocLinks !== 37) failures.push("36 个编号专题加阅读链的目录结构异常");
if (overview.ledgers !== 32 || overview.reportMap !== 9) failures.push("32 张问题账或报告地图异常");
if (overview.figureAtlas !== 21 || overview.paperLinks !== 100 || overview.paperGroups < 12) failures.push("图表审计或 100 节点阅读链异常");
if (overview.labTabs !== 8 || overview.labPanels !== 8) failures.push("八联实验结构异常");
if (overview.artifactTabs !== 4 || overview.artifactPanels !== 4 || overview.artifactLayers !== 93 || !overview.artifactMismatch) failures.push("开放工件四视图、93 层条带或形状冲突异常");
if (overview.attnresTabs !== 5 || overview.attnresPanels !== 5) failures.push("AttnRes 独立实验五视图异常");
if (overview.gradientTabs !== 5 || overview.gradientPanels !== 5) failures.push("AttnRes 梯度定义扩展五视图异常");
if (overview.spikeTabs !== 5 || overview.spikePanels !== 5) failures.push("AttnRes 尖峰路径五视图异常");
if (overview.localPathTabs !== 5 || overview.localPathPanels !== 5 || !overview.localPathVerdict) failures.push("AttnRes 局部路径五视图或冻结判定异常");
if (overview.forwardTabs !== 5 || overview.forwardPanels !== 5 || !overview.forwardVerdict) failures.push("AttnRes 训练期前向五视图或冻结判定异常");
if (!overview.nativeVisionCorrected || overview.staleVisionClaim) failures.push("原生多模态纠错未生效或旧错误残留");
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在文档级横向溢出");
if (labs.memoryInitial.panel !== "memory" || numeric(labs.memoryInitial.additiveError) <= numeric(labs.memoryInitial.deltaError)) failures.push("Delta memory 初始递推异常");
if (numeric(labs.memoryChanged.additive) <= numeric(labs.memoryInitial.additive) || numeric(labs.memoryChanged.delta) <= numeric(labs.memoryInitial.delta)) failures.push("Delta memory 控件未更新");
if (labs.decayInitial.panel !== "decay" || labs.decayInitial.verdict !== "WITHIN RANGE" || numeric(labs.decayInitial.log) !== -80) failures.push("K3 默认 bounded decay 范围异常");
if (labs.decayRisk.verdict !== "OVERFLOW RISK" || numeric(labs.decayRisk.log) !== -384) failures.push("bounded decay 风险探针异常");
if (numeric(labs.depthInitial.sources) !== 9 || numeric(labs.depthChanged.sources) !== 17) failures.push("Block AttnRes 来源计数异常");
if (numeric(labs.widthInitial.reduction) !== 50 || numeric(labs.widthFull.reduction) !== 0) failures.push("LatentMoE payload 账异常");
if (numeric(labs.situInitial.bound) !== 100 || numeric(labs.situLarge.value) > 100.01 || numeric(labs.situLarge.swiglu) <= 1000) failures.push("SiTU formal bound 异常");
if (!labs.qbInitial.before.includes("4, 3, 1, 0") || numeric(labs.qbInitial.gap) > 1 || numeric(labs.qbOff.gap) < 3) failures.push("Quantile Balancing toy route 异常");
if (!labs.rlInitial.complete.includes("192") || !labs.rlInitial.paused.includes("64") || numeric(labs.rlInitial.reward) !== 0.693) failures.push("MOPD / partial rollout 初始账异常");
if (!labs.rlOver.budget.includes("reward 改为 −1")) failures.push("Reasoning effort 超预算未触发");
if (numeric(labs.cacheInitial.hit) !== 2560 || !labs.cacheInitial.recompute.includes("256")) failures.push("Hybrid prefix cache 默认命中异常");
if (numeric(labs.cacheSparse.hit) >= numeric(labs.cacheInitial.hit) || numeric(labs.cacheSparse.recompute) <= numeric(labs.cacheInitial.recompute)) failures.push("稀疏 KDA checkpoint 未降低 joint hit");
if (labs.keyboardSelected !== "decay" || labs.keyboardVisible !== "decay") failures.push("实验键盘 tab 导航异常");
if (artifacts.initial.panel !== "layers" || numeric(artifacts.initial.layer) !== 1 || artifacts.initial.attention !== "KDA" || artifacts.initial.ffn !== "DENSE") failures.push("开放工件初始层视图异常");
if (numeric(artifacts.terminal.layer) !== 93 || artifacts.terminal.attention !== "MLA" || artifacts.terminal.ffn !== "MOE" || !artifacts.terminal.copy.includes("L92 / L93")) failures.push("K3 末层真实配置条带异常");
if (artifacts.tensors.panel !== "tensors" || artifacts.tensors.groups !== 3 || artifacts.tensors.expertRows !== 6 || !artifacts.tensors.entries || !artifacts.tensors.share) failures.push("checkpoint tensor anatomy 异常");
if (!artifacts.mla.visible || artifacts.mla.rows !== 5 || !artifacts.mla.has576) failures.push("MLA header shape 视图异常");
if (artifacts.parameterInitial.panel !== "parameters" || artifacts.parameterInitial.shape !== "[128] F32" || !artifacts.parameterInitial.conflict) failures.push("A_log 工件冲突审计异常");
if (artifacts.parameterChanged.shape !== "[96,128] F32" || !artifacts.parameterChanged.count.includes("12,288")) failures.push("真实 dt_bias 参数切换异常");
if (artifacts.reproductionInitial.panel !== "reproduction" || numeric(artifacts.reproductionInitial.speedup) !== 1.85 || numeric(artifacts.reproductionInitial.localMean) < 2.6 || !artifacts.reproductionInitial.exactSuite || numeric(artifacts.reproductionInitial.cv) < 2) failures.push("FlashKDA H20、本机 exact suite 或 router 初始探针异常");
if (numeric(artifacts.reproductionChanged.speedup) !== 3.27 || numeric(artifacts.reproductionChanged.flash) !== 0.7064 || numeric(artifacts.reproductionChanged.localMean) >= numeric(artifacts.reproductionInitial.localMean) || !artifacts.reproductionChanged.localMode.includes("FP32 state") || numeric(artifacts.reproductionChanged.cv) <= numeric(artifacts.reproductionInitial.cv) || numeric(artifacts.reproductionChanged.zero) <= numeric(artifacts.reproductionInitial.zero)) failures.push("GB200 benchmark、本机 varlen/state 或 synthetic router counterexample 未更新");
if (artifacts.keyboardSelected !== "tensors" || artifacts.keyboardVisible !== "tensors") failures.push("开放工件键盘 tab 导航异常");
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 8 || mobile.artifactTabs !== 4 || mobile.artifactLayers !== 93 || mobile.attnresTabs !== 5 || mobile.gradientTabs !== 5 || mobile.spikeTabs !== 5 || mobile.localPathTabs !== 5 || mobile.forwardTabs !== 5) failures.push("移动端导航或实验异常");
if (mobile.offenders.length) failures.push(`移动端越界元素:${JSON.stringify(mobile.offenders)}`);
if (exceptions.length) failures.push(`浏览器异常:${exceptions.join(" | ")}`);
if (failures.length) {
console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
process.exitCode = 1;
} else {
console.log("\nPASS K3 browser regression");
}
socket.close();