wuyang
303 lines
13 KiB
JavaScript
303 lines
13 KiB
JavaScript
import { writeFileSync } from "node:fs";
|
||
|
||
const cdpPort = process.env.CDP_PORT ?? "9231";
|
||
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
|
||
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`)
|
||
.then((response) => response.json());
|
||
const page = pages.find((entry) => entry.type === "page");
|
||
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
|
||
|
||
const socket = new WebSocket(page.webSocketDebuggerUrl);
|
||
await new Promise((resolve, reject) => {
|
||
socket.addEventListener("open", resolve, { once: true });
|
||
socket.addEventListener("error", reject, { once: true });
|
||
});
|
||
|
||
let nextId = 0;
|
||
const pending = new Map();
|
||
const exceptions = [];
|
||
socket.addEventListener("message", (event) => {
|
||
const message = JSON.parse(event.data);
|
||
if (message.id && pending.has(message.id)) {
|
||
const { resolve, reject } = pending.get(message.id);
|
||
pending.delete(message.id);
|
||
if (message.error) reject(new Error(message.error.message));
|
||
else resolve(message.result);
|
||
}
|
||
if (message.method === "Runtime.exceptionThrown") {
|
||
exceptions.push(
|
||
message.params.exceptionDetails.exception?.description
|
||
?? message.params.exceptionDetails.text,
|
||
);
|
||
}
|
||
});
|
||
|
||
const command = (method, params = {}) => new Promise((resolve, reject) => {
|
||
const id = ++nextId;
|
||
pending.set(id, { resolve, reject });
|
||
socket.send(JSON.stringify({ id, method, params }));
|
||
});
|
||
const pause = (milliseconds) => new Promise(
|
||
(resolve) => setTimeout(resolve, milliseconds),
|
||
);
|
||
const evaluate = async (expression) => {
|
||
const result = await command("Runtime.evaluate", {
|
||
expression,
|
||
returnByValue: true,
|
||
awaitPromise: true,
|
||
});
|
||
if (result.exceptionDetails) {
|
||
throw new Error(
|
||
result.exceptionDetails.exception?.description
|
||
?? result.exceptionDetails.text,
|
||
);
|
||
}
|
||
return result.result.value;
|
||
};
|
||
const screenshot = async (path) => {
|
||
const result = await command("Page.captureScreenshot", {
|
||
format: "png",
|
||
captureBeyondViewport: false,
|
||
});
|
||
writeFileSync(path, Buffer.from(result.data, "base64"));
|
||
};
|
||
const assert = (condition, message) => {
|
||
if (!condition) throw new Error(message);
|
||
};
|
||
|
||
await command("Page.enable");
|
||
await command("Runtime.enable");
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 1440,
|
||
height: 1100,
|
||
deviceScaleFactor: 1,
|
||
mobile: false,
|
||
});
|
||
await command("Page.navigate", {
|
||
url: `${baseUrl}/deepseek/#task-bootstrap-crn`,
|
||
});
|
||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||
await pause(100);
|
||
if (await evaluate("document.readyState === 'complete'")) break;
|
||
}
|
||
|
||
const overview = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
if (!root) return null;
|
||
document.documentElement.style.scrollBehavior = "auto";
|
||
window.scrollTo(0, root.getBoundingClientRect().top + window.scrollY);
|
||
const facts = Object.fromEntries(
|
||
[...root.querySelectorAll(".tb-ledger article")].map((node) => [
|
||
node.querySelector("span").textContent.trim(),
|
||
node.querySelector("b").textContent.trim(),
|
||
]),
|
||
);
|
||
return {
|
||
tabs: root.querySelectorAll("[data-tb-tab]").length,
|
||
panels: root.querySelectorAll("[data-tb-panel]").length,
|
||
active: root.querySelector("[data-tb-panel]:not([hidden])")?.dataset.tbPanel,
|
||
bars: root.querySelectorAll(".uniform-bars article").length,
|
||
lanes: root.querySelectorAll(".token-lanes article").length,
|
||
laneTokens: root.querySelectorAll(".token-lanes article i").length,
|
||
facts,
|
||
labs: [...document.querySelectorAll(".page-facts > div")]
|
||
.find((node) => node.querySelector("dt")?.textContent.trim() === "LABS")
|
||
?.querySelector("dd")?.textContent.trim(),
|
||
status: [...document.querySelectorAll(".page-facts > div")]
|
||
.find((node) => node.querySelector("dt")?.textContent.trim() === "STATUS")
|
||
?.querySelector("dd")?.textContent.trim(),
|
||
documentOverflow: document.documentElement.scrollWidth
|
||
- document.documentElement.clientWidth,
|
||
};
|
||
})()`);
|
||
await pause(250);
|
||
await screenshot("/tmp/llm-atlas-task-bootstrap-sampler-desktop.png");
|
||
|
||
const bootstrap = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
root.querySelector('[data-tb-tab="bootstrap"]').click();
|
||
const read = () => ({
|
||
active: root.querySelector("[data-tb-panel]:not([hidden])").dataset.tbPanel,
|
||
conditionCards: [...root.querySelectorAll("[data-tb-condition-cards] article")]
|
||
.map((node) => node.querySelector("b").textContent.trim()),
|
||
forest: [...root.querySelectorAll("[data-tb-forest] > article")].map((node) => ({
|
||
label: node.querySelector("span").textContent.trim(),
|
||
point: node.querySelector(":scope > b").textContent.trim(),
|
||
band: node.querySelector("small").textContent.trim(),
|
||
clear: node.querySelector("u").classList.contains("clear"),
|
||
})),
|
||
robust: root.querySelector("[data-tb-robust-count]").textContent.trim(),
|
||
});
|
||
const codeCorrectness = read();
|
||
root.querySelector('[data-tb-metric="generated_tokens"]').click();
|
||
const codeLength = read();
|
||
root.querySelector('[data-tb-domain="math"]').click();
|
||
const mathLength = read();
|
||
return { codeCorrectness, codeLength, mathLength };
|
||
})()`);
|
||
await pause(150);
|
||
await screenshot("/tmp/llm-atlas-task-bootstrap-forest-desktop.png");
|
||
|
||
const taskExplorer = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
root.querySelector('[data-tb-tab="tasks"]').click();
|
||
const domain = root.querySelector("[data-tb-task-domain]");
|
||
const contrast = root.querySelector("[data-tb-task-contrast]");
|
||
domain.value = "math";
|
||
domain.dispatchEvent(new Event("change", { bubbles: true }));
|
||
contrast.value = "system_at_eos";
|
||
contrast.dispatchEvent(new Event("change", { bubbles: true }));
|
||
root.querySelector('[data-tb-task-page="2"]').click();
|
||
return {
|
||
active: root.querySelector("[data-tb-panel]:not([hidden])").dataset.tbPanel,
|
||
rows: root.querySelectorAll("[data-tb-task-rows] > article").length,
|
||
page: root.querySelector("[data-tb-task-page-label]").textContent.trim(),
|
||
selectedPage: root.querySelector('[data-tb-task-page][aria-pressed="true"]')
|
||
.dataset.tbTaskPage,
|
||
firstTask: root.querySelector("[data-tb-task-rows] > article span b")
|
||
.textContent.trim(),
|
||
transition: [...root.querySelectorAll("[data-tb-transition-cards] article")]
|
||
.map((node) => node.querySelector("b").textContent.trim()),
|
||
};
|
||
})()`);
|
||
|
||
const tape = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
root.querySelector('[data-tb-tab="tapes"]').click();
|
||
const domain = root.querySelector("[data-tb-tape-domain]");
|
||
const contrast = root.querySelector("[data-tb-tape-contrast]");
|
||
domain.value = "math";
|
||
domain.dispatchEvent(new Event("change", { bubbles: true }));
|
||
contrast.value = "system_at_eos";
|
||
contrast.dispatchEvent(new Event("change", { bubbles: true }));
|
||
return {
|
||
active: root.querySelector("[data-tb-panel]:not([hidden])").dataset.tbPanel,
|
||
rows: root.querySelectorAll("[data-tb-tape-rows] > article").length,
|
||
cells: root.querySelectorAll("[data-tb-tape-rows] > article > b").length,
|
||
means: [...root.querySelectorAll("[data-tb-tape-means] > b")]
|
||
.map((node) => node.textContent.trim()),
|
||
taskRange: root.querySelector("[data-tb-task-range]").textContent.trim(),
|
||
tapeRange: root.querySelector("[data-tb-tape-range]").textContent.trim(),
|
||
};
|
||
})()`);
|
||
|
||
const audit = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
root.querySelector('[data-tb-tab="audit"]').click();
|
||
return {
|
||
active: root.querySelector("[data-tb-panel]:not([hidden])").dataset.tbPanel,
|
||
fields: [...root.querySelectorAll(".replay-fields article")]
|
||
.map((node) => node.querySelector("b").textContent.trim()),
|
||
codeOutcomes: root.querySelectorAll(".failure-ledger > article:first-child > div").length,
|
||
mathOutcomes: root.querySelectorAll(".failure-ledger > article:last-child > div").length,
|
||
deviations: root.querySelectorAll(".deviation-ledger article").length,
|
||
artifacts: root.querySelectorAll(".artifact-chain article").length,
|
||
};
|
||
})()`);
|
||
|
||
const keyboard = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
const first = root.querySelector('[data-tb-tab="sampler"]');
|
||
first.click();
|
||
first.focus();
|
||
first.dispatchEvent(new KeyboardEvent("keydown", {
|
||
key: "ArrowRight",
|
||
bubbles: true,
|
||
}));
|
||
return {
|
||
selected: root.querySelector('[data-tb-tab][aria-selected="true"]').dataset.tbTab,
|
||
active: root.querySelector("[data-tb-panel]:not([hidden])").dataset.tbPanel,
|
||
focused: document.activeElement.dataset.tbTab,
|
||
};
|
||
})()`);
|
||
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 390,
|
||
height: 844,
|
||
deviceScaleFactor: 1,
|
||
mobile: true,
|
||
});
|
||
await pause(300);
|
||
const mobile = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-task-bootstrap-lab]");
|
||
root.querySelector('[data-tb-tab="tasks"]').click();
|
||
root.scrollIntoView();
|
||
return {
|
||
documentOverflow: document.documentElement.scrollWidth
|
||
- document.documentElement.clientWidth,
|
||
rootOverflow: root.scrollWidth - root.clientWidth,
|
||
tableWidth: root.querySelector(".task-table").getBoundingClientRect().width,
|
||
tableScrollWidth: root.querySelector(".task-table").scrollWidth,
|
||
viewport: window.innerWidth,
|
||
};
|
||
})()`);
|
||
await screenshot("/tmp/llm-atlas-task-bootstrap-mobile.png");
|
||
|
||
assert(overview, "找不到任务 bootstrap 实验");
|
||
assert(overview.tabs === 5 && overview.panels === 5, "五页签/面板合同异常");
|
||
assert(overview.active === "sampler", "初始面板不是 sampler");
|
||
assert(overview.bars === 8, "uniform tape 前八步渲染异常");
|
||
assert(overview.lanes === 4 && overview.laneTokens === 32, "四条件 token lane 异常");
|
||
assert(overview.facts.TASKS === "32 + 32", "任务 headline 异常");
|
||
assert(overview.facts["FORMAL GRID"] === "352", "formal headline 异常");
|
||
assert(overview.facts["UNIFORM AUDIT"] === "352 / 352", "uniform headline 异常");
|
||
assert(overview.facts["FRESH PROCESS"] === "64 / 64", "replay headline 异常");
|
||
assert(overview.labs === "22 个可操作实验", "DeepSeek LABS 总账异常");
|
||
assert(overview.status === "八轮 · 864 条采样", "DeepSeek STATUS 总账异常");
|
||
assert(overview.documentOverflow <= 1, `桌面横向溢出 ${overview.documentOverflow}px`);
|
||
|
||
assert(bootstrap.codeCorrectness.active === "bootstrap", "bootstrap 面板切换异常");
|
||
assert(bootstrap.codeCorrectness.conditionCards.join("|") === "16 / 32 pass|16 / 32 pass|15 / 32 pass|12 / 32 pass", "Code 条件 pass 表异常");
|
||
assert(bootstrap.codeCorrectness.forest.length === 4, "forest contrast 数异常");
|
||
assert(bootstrap.codeCorrectness.robust === "0 / 4 bands 不跨 0", "Code correctness 带数异常");
|
||
assert(bootstrap.codeLength.robust === "2 / 4 bands 不跨 0", "Code length 带数异常");
|
||
assert(bootstrap.codeLength.forest.at(-1).point === "-130.9 tok", "Code system-period 长度异常");
|
||
assert(bootstrap.mathLength.robust === "1 / 4 bands 不跨 0", "Math length 带数异常");
|
||
assert(bootstrap.mathLength.forest.at(-1).point === "+25.1 tok", "Math system-period 长度异常");
|
||
|
||
assert(taskExplorer.active === "tasks", "逐题面板切换异常");
|
||
assert(taskExplorer.rows === 8 && taskExplorer.page === "17–24 / 32", "逐题分页异常");
|
||
assert(taskExplorer.selectedPage === "2", "逐题页按钮状态异常");
|
||
assert(taskExplorer.firstTask === "gsm8k/test/1050", "逐题页首任务异常");
|
||
assert(taskExplorer.transition.join("|") === "3 / 32|3 / 32|16 / 32|10 / 32", "Math system-EOS 转移异常");
|
||
|
||
assert(tape.active === "tapes", "tape 面板切换异常");
|
||
assert(tape.rows === 4 && tape.cells === 16, "4×4 tape matrix 异常");
|
||
assert(tape.means.join("|") === "0.00|+0.50|-0.25|+0.25", "Math system-EOS tape means 异常");
|
||
assert(tape.taskRange === "1.25" && tape.tapeRange === "1.00", "tape/task range 异常");
|
||
|
||
assert(audit.active === "audit", "审计面板切换异常");
|
||
assert(audit.fields.length === 12, "重放字段数不为 12");
|
||
assert(audit.fields.every((value) => value === "64 / 64"), "重放字段未全部 exact");
|
||
assert(audit.codeOutcomes === 6 && audit.mathOutcomes === 2, "失败分类数量异常");
|
||
assert(audit.deviations === 2, "偏离账数量异常");
|
||
assert(audit.artifacts === 4, "工件 hash 数量异常");
|
||
assert(
|
||
keyboard.selected === "bootstrap"
|
||
&& keyboard.active === "bootstrap"
|
||
&& keyboard.focused === "bootstrap",
|
||
"页签键盘导航异常",
|
||
);
|
||
assert(mobile.documentOverflow <= 1, `移动端 document 横向溢出 ${mobile.documentOverflow}px`);
|
||
assert(mobile.rootOverflow <= 1, `移动端实验横向溢出 ${mobile.rootOverflow}px`);
|
||
assert(mobile.tableWidth <= mobile.viewport, "移动端任务表容器超出 viewport");
|
||
assert(mobile.tableScrollWidth > mobile.tableWidth, "移动端任务表没有内部横向滚动");
|
||
assert(exceptions.length === 0, `浏览器异常：${exceptions.join(" | ")}`);
|
||
|
||
console.log(JSON.stringify({
|
||
overview,
|
||
bootstrap,
|
||
taskExplorer,
|
||
tape,
|
||
audit,
|
||
keyboard,
|
||
mobile,
|
||
screenshots: [
|
||
"/tmp/llm-atlas-task-bootstrap-sampler-desktop.png",
|
||
"/tmp/llm-atlas-task-bootstrap-forest-desktop.png",
|
||
"/tmp/llm-atlas-task-bootstrap-mobile.png",
|
||
],
|
||
}, null, 2));
|
||
|
||
socket.close();
|