wuyang
201 lines
9.3 KiB
JavaScript
201 lines
9.3 KiB
JavaScript
import { writeFileSync } from "node:fs";
|
||
|
||
const cdpPort = process.env.CDP_PORT ?? "9227";
|
||
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
|
||
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
|
||
const page = pages.find((entry) => entry.type === "page");
|
||
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
|
||
|
||
const socket = new WebSocket(page.webSocketDebuggerUrl);
|
||
await new Promise((resolve, reject) => {
|
||
socket.addEventListener("open", resolve, { once: true });
|
||
socket.addEventListener("error", reject, { once: true });
|
||
});
|
||
|
||
let nextId = 0;
|
||
const pending = new Map();
|
||
const exceptions = [];
|
||
socket.addEventListener("message", (event) => {
|
||
const message = JSON.parse(event.data);
|
||
if (message.id && pending.has(message.id)) {
|
||
const { resolve, reject } = pending.get(message.id);
|
||
pending.delete(message.id);
|
||
if (message.error) reject(new Error(message.error.message));
|
||
else resolve(message.result);
|
||
}
|
||
if (message.method === "Runtime.exceptionThrown") {
|
||
exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
|
||
}
|
||
});
|
||
|
||
const command = (method, params = {}) => new Promise((resolve, reject) => {
|
||
const id = ++nextId;
|
||
pending.set(id, { resolve, reject });
|
||
socket.send(JSON.stringify({ id, method, params }));
|
||
});
|
||
const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||
const evaluate = async (expression) => {
|
||
const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
|
||
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
|
||
return result.result.value;
|
||
};
|
||
const navigate = async (path) => {
|
||
await command("Page.navigate", { url: `${baseUrl}${path}` });
|
||
for (let attempt = 0; attempt < 80; attempt += 1) {
|
||
await pause(100);
|
||
if (await evaluate("document.readyState === 'complete'")) return;
|
||
}
|
||
throw new Error(`${path} 加载超时`);
|
||
};
|
||
const screenshot = async (path) => {
|
||
const result = await command("Page.captureScreenshot", { format: "png", captureBeyondViewport: false });
|
||
writeFileSync(path, Buffer.from(result.data, "base64"));
|
||
};
|
||
|
||
await command("Page.enable");
|
||
await command("Runtime.enable");
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 1440,
|
||
height: 1100,
|
||
deviceScaleFactor: 1,
|
||
mobile: false,
|
||
});
|
||
await navigate("/k3/");
|
||
|
||
const desktop = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-attnres-lab]");
|
||
root.scrollIntoView({ block: "start", behavior: "instant" });
|
||
window.scrollBy(0, -78);
|
||
const text = (selector) => root.querySelector(selector)?.textContent.trim();
|
||
const panel = () => root.querySelector("[data-attnres-panel]:not([hidden])")?.dataset.attnresPanel;
|
||
const line = (architecture) => root.querySelector(\`[data-curve-series="\${architecture}"] [data-curve-line]\`)?.getAttribute("points");
|
||
|
||
const initial = {
|
||
panel: panel(),
|
||
tabs: root.querySelectorAll("[data-attnres-tab]").length,
|
||
panels: root.querySelectorAll("[data-attnres-panel]").length,
|
||
baseline: text('[data-curve-final="baseline"]'),
|
||
full: text('[data-curve-final="full"]'),
|
||
block: text('[data-curve-final="block"]'),
|
||
baselineLine: line("baseline"),
|
||
textBoundary: root.textContent.includes("不是 K3 checkpoint forward") &&
|
||
root.textContent.includes("不能写成论文梯度结果复现"),
|
||
};
|
||
|
||
root.querySelector('[data-curve-seed="0"]').click();
|
||
const seed = {
|
||
baseline: text('[data-curve-final="baseline"]'),
|
||
full: text('[data-curve-final="full"]'),
|
||
block: text('[data-curve-final="block"]'),
|
||
baselineLine: line("baseline"),
|
||
};
|
||
|
||
root.querySelector('[data-attnres-tab="rms"]').click();
|
||
const rmsInitial = {
|
||
panel: panel(),
|
||
title: text("[data-rms-title]"),
|
||
blockLine: root.querySelector('[data-rms-series="block"]').getAttribute("points"),
|
||
rhythm: root.querySelectorAll(".block-rhythm i").length,
|
||
};
|
||
root.querySelector('[data-rms-metric="branch_output_rms"]').click();
|
||
const rmsChanged = {
|
||
title: text("[data-rms-title]"),
|
||
blockLine: root.querySelector('[data-rms-series="block"]').getAttribute("points"),
|
||
};
|
||
|
||
root.querySelector('[data-attnres-tab="mixer"]').click();
|
||
const mixerInitial = {
|
||
panel: panel(),
|
||
fullVisible: !root.querySelector('[data-mixer-view="full"]').hidden,
|
||
fullRows: root.querySelectorAll('[data-mixer-view="full"] .heat-row').length,
|
||
spike: root.querySelectorAll(".spikeSource").length,
|
||
};
|
||
root.querySelector('[data-mixer-arch="block"]').click();
|
||
const mixerChanged = {
|
||
blockVisible: !root.querySelector('[data-mixer-view="block"]').hidden,
|
||
blockRows: root.querySelectorAll('[data-mixer-view="block"] .heat-row').length,
|
||
};
|
||
|
||
root.querySelector('[data-attnres-tab="gradient"]').click();
|
||
const gradient = {
|
||
panel: panel(),
|
||
cards: root.querySelectorAll(".gradient-cv article").length,
|
||
blocks: root.querySelectorAll(".gradient-grid article").length,
|
||
boundary: root.textContent.includes("BASE < FULL < BLOCK") &&
|
||
root.textContent.includes("不能写成论文梯度结果复现"),
|
||
};
|
||
|
||
root.querySelector('[data-attnres-tab="audit"]').click();
|
||
const audit = {
|
||
panel: panel(),
|
||
costs: root.querySelectorAll(".cost-grid article").length,
|
||
hashes: root.querySelectorAll(".hash-ledger code").length,
|
||
exact: root.textContent.includes("2,000 steps · 8 / 8 exact"),
|
||
claims: root.querySelectorAll(".claim-grid li").length,
|
||
};
|
||
|
||
const first = root.querySelector('[data-attnres-tab="outcome"]');
|
||
first.focus();
|
||
first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
||
const keyboard = {
|
||
selected: root.querySelector('[data-attnres-tab][aria-selected="true"]').dataset.attnresTab,
|
||
panel: panel(),
|
||
};
|
||
|
||
return {
|
||
initial, seed, rmsInitial, rmsChanged, mixerInitial, mixerChanged, gradient, audit, keyboard,
|
||
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||
};
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-k3-attnres-desktop.png");
|
||
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 390,
|
||
height: 844,
|
||
deviceScaleFactor: 1,
|
||
mobile: true,
|
||
});
|
||
await navigate("/k3/");
|
||
const mobile = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-attnres-lab]");
|
||
root.scrollIntoView({ block: "start", behavior: "instant" });
|
||
window.scrollBy(0, -64);
|
||
return {
|
||
tabs: root.querySelectorAll("[data-attnres-tab]").length,
|
||
ledger: root.querySelectorAll(".trace-ledger article").length,
|
||
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||
rootOverflow: root.scrollWidth - root.clientWidth,
|
||
visiblePanel: root.querySelector("[data-attnres-panel]:not([hidden])")?.dataset.attnresPanel,
|
||
};
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-k3-attnres-mobile.png");
|
||
|
||
const report = { desktop, mobile, exceptions };
|
||
console.log(JSON.stringify(report, null, 2));
|
||
|
||
const numeric = (text) => Number.parseFloat(text.replace("−", "-"));
|
||
const failures = [];
|
||
if (desktop.initial.panel !== "outcome" || desktop.initial.tabs !== 5 || desktop.initial.panels !== 5) failures.push("五视图初始结构异常");
|
||
if (Math.abs(numeric(desktop.initial.baseline) - 1.99913) > 1e-5 || Math.abs(numeric(desktop.initial.full) - 1.98457) > 1e-5 || Math.abs(numeric(desktop.initial.block) - 1.95667) > 1e-5) failures.push("三 seed 均值 BPC 异常");
|
||
if (Math.abs(numeric(desktop.seed.baseline) - 2.00054) > 1e-5 || desktop.seed.baselineLine === desktop.initial.baselineLine) failures.push("seed 切换未更新训练曲线");
|
||
if (!desktop.initial.textBoundary) failures.push("主视区缺少 K3 forward 或梯度反证边界");
|
||
if (desktop.rmsInitial.panel !== "rms" || desktop.rmsInitial.rhythm !== 32 || desktop.rmsInitial.title === desktop.rmsChanged.title || desktop.rmsInitial.blockLine === desktop.rmsChanged.blockLine) failures.push("RMS 指标切换或 32 层节律异常");
|
||
if (desktop.mixerInitial.panel !== "mixer" || !desktop.mixerInitial.fullVisible || desktop.mixerInitial.fullRows !== 32 || desktop.mixerInitial.spike !== 1 || !desktop.mixerChanged.blockVisible || desktop.mixerChanged.blockRows !== 32) failures.push("Full/Block mixer heatmap 异常");
|
||
if (desktop.gradient.panel !== "gradient" || desktop.gradient.cards !== 4 || desktop.gradient.blocks !== 16 || !desktop.gradient.boundary) failures.push("梯度反证视图异常");
|
||
if (desktop.audit.panel !== "audit" || desktop.audit.costs !== 3 || desktop.audit.hashes !== 4 || !desktop.audit.exact || desktop.audit.claims !== 6) failures.push("成本、重放或 claim boundary 异常");
|
||
if (desktop.keyboard.selected !== "rms" || desktop.keyboard.panel !== "rms") failures.push("键盘 tab 导航异常");
|
||
if (desktop.documentOverflow > 1 || mobile.documentOverflow > 1 || mobile.rootOverflow > 1) failures.push("桌面或移动端出现文档级横向溢出");
|
||
if (mobile.tabs !== 5 || mobile.ledger !== 6 || mobile.visiblePanel !== "outcome") failures.push("移动端初始结构异常");
|
||
if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);
|
||
|
||
if (failures.length) {
|
||
console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
|
||
process.exitCode = 1;
|
||
} else {
|
||
console.log("\nPASS K3 AttnRes browser regression");
|
||
}
|
||
|
||
socket.close();
|