wuyang
252 lines
12 KiB
JavaScript
252 lines
12 KiB
JavaScript
import { writeFileSync } from "node:fs";
|
||
|
||
const cdpPort = process.env.CDP_PORT ?? "9224";
|
||
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4323";
|
||
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
|
||
const page = pages.find((entry) => entry.type === "page");
|
||
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
|
||
|
||
const socket = new WebSocket(page.webSocketDebuggerUrl);
|
||
await new Promise((resolve, reject) => {
|
||
socket.addEventListener("open", resolve, { once: true });
|
||
socket.addEventListener("error", reject, { once: true });
|
||
});
|
||
|
||
let nextId = 0;
|
||
const pending = new Map();
|
||
const exceptions = [];
|
||
socket.addEventListener("message", (event) => {
|
||
const message = JSON.parse(event.data);
|
||
if (message.id && pending.has(message.id)) {
|
||
const { resolve, reject } = pending.get(message.id);
|
||
pending.delete(message.id);
|
||
if (message.error) reject(new Error(message.error.message));
|
||
else resolve(message.result);
|
||
}
|
||
if (message.method === "Runtime.exceptionThrown") {
|
||
exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
|
||
}
|
||
});
|
||
|
||
const command = (method, params = {}) => new Promise((resolve, reject) => {
|
||
const id = ++nextId;
|
||
pending.set(id, { resolve, reject });
|
||
socket.send(JSON.stringify({ id, method, params }));
|
||
});
|
||
const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||
const evaluate = async (expression) => {
|
||
const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
|
||
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
|
||
return result.result.value;
|
||
};
|
||
const navigate = async (path) => {
|
||
await command("Page.navigate", { url: `${baseUrl}${path}` });
|
||
for (let attempt = 0; attempt < 70; attempt += 1) {
|
||
await pause(100);
|
||
if (await evaluate("document.readyState === 'complete'")) return;
|
||
}
|
||
throw new Error(`${path} 加载超时`);
|
||
};
|
||
const screenshot = async (path, full = false) => {
|
||
const params = { format: "png", captureBeyondViewport: full };
|
||
if (full) {
|
||
const metrics = await command("Page.getLayoutMetrics");
|
||
params.clip = {
|
||
x: 0,
|
||
y: 0,
|
||
width: metrics.cssContentSize.width,
|
||
height: metrics.cssContentSize.height,
|
||
scale: 1,
|
||
};
|
||
}
|
||
const result = await command("Page.captureScreenshot", params);
|
||
writeFileSync(path, Buffer.from(result.data, "base64"));
|
||
};
|
||
|
||
await command("Page.enable");
|
||
await command("Runtime.enable");
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 1440,
|
||
height: 1100,
|
||
deviceScaleFactor: 1,
|
||
mobile: false,
|
||
});
|
||
await navigate("/foundations/");
|
||
await screenshot("/tmp/llm-atlas-transformer-desktop.png");
|
||
|
||
const overview = await evaluate(`(() => ({
|
||
title: document.querySelector("h1")?.textContent.trim(),
|
||
sections: document.querySelectorAll(".article-section").length,
|
||
tocLinks: document.querySelectorAll(".side-rail a").length,
|
||
paperLinks: document.querySelectorAll(".paper-chain a").length,
|
||
labTabs: document.querySelectorAll("[data-attention-tab]").length,
|
||
labPanels: document.querySelectorAll("[data-attention-panel]").length,
|
||
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||
}))()`);
|
||
|
||
const qkv = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-transformer-lab]");
|
||
const read = () => ({
|
||
output: root.querySelector("[data-qkv-output]").textContent,
|
||
dimension: root.querySelector("[data-qkv-dimension-label]").textContent,
|
||
temperature: root.querySelector("[data-qkv-temperature-label]").textContent,
|
||
weights: [...root.querySelectorAll("[data-qkv-weight]")].map((node) => node.textContent),
|
||
});
|
||
const initial = read();
|
||
const dimension = root.querySelector("[data-qkv-dimension]");
|
||
dimension.value = "4";
|
||
dimension.dispatchEvent(new Event("input", { bubbles: true }));
|
||
const temperature = root.querySelector("[data-qkv-temperature]");
|
||
temperature.value = "250";
|
||
temperature.dispatchEvent(new Event("input", { bubbles: true }));
|
||
const sharp = read();
|
||
return { initial, sharp };
|
||
})()`);
|
||
|
||
const mask = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-transformer-lab]");
|
||
root.querySelector('[data-attention-tab="mask"]').click();
|
||
root.querySelector('[data-mask-mode="prefix"]').click();
|
||
root.querySelector('[data-mask-query="0"]').click();
|
||
const prefix = {
|
||
label: root.querySelector("[data-mask-label]").textContent,
|
||
sources: root.querySelector("[data-mask-sources]").textContent,
|
||
allowed: root.querySelectorAll(".mask-grid i.allowed.selected-row").length,
|
||
};
|
||
root.querySelector('[data-mask-mode="causal"]').click();
|
||
root.querySelector('[data-mask-query="2"]').click();
|
||
const causal = {
|
||
sources: root.querySelector("[data-mask-sources]").textContent,
|
||
allowed: root.querySelectorAll(".mask-grid i.allowed.selected-row").length,
|
||
visiblePanel: root.querySelector("[data-attention-panel]:not([hidden])").dataset.attentionPanel,
|
||
};
|
||
return { prefix, causal };
|
||
})()`);
|
||
|
||
const heads = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-transformer-lab]");
|
||
root.querySelector('[data-attention-tab="heads"]').click();
|
||
const select = root.querySelector("[data-position-mode]");
|
||
select.value = "alibi";
|
||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||
return {
|
||
kicker: root.querySelector("[data-position-kicker]").textContent,
|
||
title: root.querySelector("[data-position-title]").textContent,
|
||
mode: root.querySelector("[data-position-visual]").dataset.mode,
|
||
localHeights: [...root.querySelectorAll('[data-head-bar^="local-"]')].map((node) => node.style.height),
|
||
};
|
||
})()`);
|
||
|
||
const block = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-transformer-lab]");
|
||
root.querySelector('[data-attention-tab="block"]').click();
|
||
root.querySelector('[data-block-id="k3"]').click();
|
||
const context = root.querySelector("[data-context-length]");
|
||
context.value = "131072";
|
||
context.dispatchEvent(new Event("input", { bubbles: true }));
|
||
const selected = {
|
||
family: root.querySelector("[data-block-family]").textContent,
|
||
kv: root.querySelector("[data-block-kv]").textContent,
|
||
path: [...root.querySelectorAll("[data-block-path] b")].map((node) => node.textContent),
|
||
note: root.querySelector("[data-block-note]").textContent,
|
||
context: root.querySelector("[data-context-label]").textContent,
|
||
mha: root.querySelector('[data-cost-value="mha"]').textContent,
|
||
kda: root.querySelector('[data-cost-value="kda"]').textContent,
|
||
};
|
||
const firstTab = root.querySelector('[data-attention-tab="qkv"]');
|
||
firstTab.focus();
|
||
firstTab.dispatchEvent(new KeyboardEvent("keydown", { key: "End", bubbles: true }));
|
||
return {
|
||
...selected,
|
||
keyboardSelected: root.querySelector('[data-attention-tab][aria-selected="true"]').dataset.attentionTab,
|
||
keyboardVisible: root.querySelector("[data-attention-panel]:not([hidden])").dataset.attentionPanel,
|
||
};
|
||
})()`);
|
||
|
||
await evaluate(`document.querySelector("[data-transformer-lab]").scrollIntoView({ block: "start", behavior: "instant" })`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-transformer-lab-desktop.png");
|
||
|
||
await navigate("/");
|
||
const home = await evaluate(`(() => ({
|
||
releaseCards: document.querySelectorAll(".release-card").length,
|
||
firstRelease: document.querySelector(".release-card h2").textContent,
|
||
firstHref: document.querySelector(".release-card").getAttribute("href"),
|
||
paperCount: [...document.querySelectorAll(".hero-stats b")].map((node) => node.textContent.trim()).find((value) => value === "258"),
|
||
}))()`);
|
||
|
||
await navigate("/papers/");
|
||
const papers = await evaluate(`(() => ({
|
||
total: document.querySelectorAll("[data-paper]").length,
|
||
transformerVisible: (() => {
|
||
const button = [...document.querySelectorAll("[data-filter]")].find((node) => node.textContent.includes("Transformer"));
|
||
button?.click();
|
||
return document.querySelectorAll("[data-paper]:not([hidden])").length;
|
||
})(),
|
||
}))()`);
|
||
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 390,
|
||
height: 844,
|
||
deviceScaleFactor: 1,
|
||
mobile: true,
|
||
});
|
||
await navigate("/foundations/");
|
||
const mobile = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-transformer-lab]");
|
||
root.scrollIntoView({ block: "start", behavior: "instant" });
|
||
const toggle = document.querySelector("#menu-toggle");
|
||
toggle?.click();
|
||
return {
|
||
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||
menuVisible: getComputedStyle(toggle).display !== "none",
|
||
menuOpen: toggle.getAttribute("aria-expanded"),
|
||
tabs: root.querySelectorAll("[data-attention-tab]").length,
|
||
offenders: [...document.querySelectorAll("body *")]
|
||
.filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
|
||
.slice(0, 12)
|
||
.map((node) => ({
|
||
tag: node.tagName,
|
||
className: typeof node.className === "string" ? node.className : "",
|
||
right: Math.round(node.getBoundingClientRect().right),
|
||
width: Math.round(node.getBoundingClientRect().width),
|
||
})),
|
||
};
|
||
})()`);
|
||
await pause(180);
|
||
await screenshot("/tmp/llm-atlas-transformer-mobile.png");
|
||
|
||
const report = { overview, qkv, mask, heads, block, home, papers, mobile, exceptions };
|
||
console.log(JSON.stringify(report, null, 2));
|
||
|
||
const numeric = (value) => Number.parseFloat(value);
|
||
const failures = [];
|
||
if (!overview.title.includes("按需取信息")) failures.push("章节标题异常");
|
||
if (overview.sections !== 22 || overview.tocLinks !== 22) failures.push("章节/目录数量异常");
|
||
if (overview.paperLinks !== 40) failures.push("正式论文链不是 40 个节点");
|
||
if (overview.labTabs !== 4 || overview.labPanels !== 4) failures.push("四联实验结构异常");
|
||
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在横向溢出");
|
||
if (qkv.initial.dimension.trim() !== "64" || qkv.sharp.dimension.trim() !== "4") failures.push("QKV 维度交互异常");
|
||
if (qkv.sharp.temperature.trim() !== "2.5×" || Math.max(...qkv.sharp.weights.map(numeric)) < 60) failures.push("QKV softmax 尖锐度没有响应");
|
||
if (mask.prefix.allowed !== 3 || !mask.prefix.sources.includes("那只")) failures.push("Prefix mask 前缀区异常");
|
||
if (mask.causal.allowed !== 3 || mask.causal.visiblePanel !== "mask") failures.push("Causal mask 可见性异常");
|
||
if (heads.mode !== "alibi" || !heads.kicker.includes("LINEAR BIASES")) failures.push("位置方案切换异常");
|
||
if (new Set(heads.localHeights).size < 2) failures.push("多头权重视图没有分化");
|
||
if (block.family.trim() !== "Hybrid MoE" || !block.kv.includes("3 KDA : 1 Gated MLA")) failures.push("K3 Block 合同异常");
|
||
if (!block.path.some((step) => step.includes("KDA × 3")) || !block.note.includes("AttnRes")) failures.push("K3 Block 路径异常");
|
||
if (block.context.trim() !== "128K" || numeric(block.mha) !== 400 || numeric(block.kda) !== 1) failures.push("KV 成本缩放异常");
|
||
if (block.keyboardSelected !== "block" || block.keyboardVisible !== "block") failures.push("实验 tab 键盘导航异常");
|
||
if (home.releaseCards !== 9 || !home.firstRelease.includes("Attention 不只是") || home.firstHref !== "/foundations/") failures.push("首页 Transformer 首发入口异常");
|
||
if (home.paperCount !== "258" || papers.total !== 258 || papers.transformerVisible < 30) failures.push("论文库或首页论文数量异常");
|
||
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4) failures.push("移动端导航或实验异常");
|
||
if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);
|
||
|
||
if (failures.length) {
|
||
console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
|
||
process.exitCode = 1;
|
||
} else {
|
||
console.log("\nPASS transformer browser regression");
|
||
}
|
||
|
||
socket.close();
|