wuyang
315 lines
13 KiB
JavaScript
315 lines
13 KiB
JavaScript
import { writeFileSync } from "node:fs";
|
||
|
||
const cdpPort = process.env.CDP_PORT ?? "9230";
|
||
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
|
||
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`)
|
||
.then((response) => response.json());
|
||
const page = pages.find((entry) => entry.type === "page");
|
||
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
|
||
|
||
const socket = new WebSocket(page.webSocketDebuggerUrl);
|
||
await new Promise((resolve, reject) => {
|
||
socket.addEventListener("open", resolve, { once: true });
|
||
socket.addEventListener("error", reject, { once: true });
|
||
});
|
||
|
||
let nextId = 0;
|
||
const pending = new Map();
|
||
const exceptions = [];
|
||
socket.addEventListener("message", (event) => {
|
||
const message = JSON.parse(event.data);
|
||
if (message.id && pending.has(message.id)) {
|
||
const { resolve, reject } = pending.get(message.id);
|
||
pending.delete(message.id);
|
||
if (message.error) reject(new Error(message.error.message));
|
||
else resolve(message.result);
|
||
}
|
||
if (message.method === "Runtime.exceptionThrown") {
|
||
exceptions.push(
|
||
message.params.exceptionDetails.exception?.description
|
||
?? message.params.exceptionDetails.text,
|
||
);
|
||
}
|
||
});
|
||
|
||
const command = (method, params = {}) => new Promise((resolve, reject) => {
|
||
const id = ++nextId;
|
||
pending.set(id, { resolve, reject });
|
||
socket.send(JSON.stringify({ id, method, params }));
|
||
});
|
||
const pause = (milliseconds) => new Promise(
|
||
(resolve) => setTimeout(resolve, milliseconds),
|
||
);
|
||
const evaluate = async (expression) => {
|
||
const result = await command("Runtime.evaluate", {
|
||
expression,
|
||
returnByValue: true,
|
||
awaitPromise: true,
|
||
});
|
||
if (result.exceptionDetails) {
|
||
throw new Error(
|
||
result.exceptionDetails.exception?.description
|
||
?? result.exceptionDetails.text,
|
||
);
|
||
}
|
||
return result.result.value;
|
||
};
|
||
const screenshot = async (path) => {
|
||
const result = await command("Page.captureScreenshot", {
|
||
format: "png",
|
||
captureBeyondViewport: false,
|
||
});
|
||
writeFileSync(path, Buffer.from(result.data, "base64"));
|
||
};
|
||
const assert = (condition, message) => {
|
||
if (!condition) throw new Error(message);
|
||
};
|
||
|
||
await command("Page.enable");
|
||
await command("Runtime.enable");
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 1440,
|
||
height: 1100,
|
||
deviceScaleFactor: 1,
|
||
mobile: false,
|
||
});
|
||
await command("Page.navigate", { url: `${baseUrl}/deepseek/` });
|
||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||
await pause(100);
|
||
if (await evaluate("document.readyState === 'complete'")) break;
|
||
}
|
||
|
||
const overview = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
if (!root) return null;
|
||
document.documentElement.style.scrollBehavior = "auto";
|
||
window.scrollTo(0, root.getBoundingClientRect().top + window.scrollY);
|
||
const facts = Object.fromEntries(
|
||
[...root.querySelectorAll(".cs-ledger article")].map((node) => [
|
||
node.querySelector("span").textContent.trim(),
|
||
node.querySelector("b").textContent.trim(),
|
||
]),
|
||
);
|
||
return {
|
||
tabs: root.querySelectorAll("[data-cs-tab]").length,
|
||
panels: root.querySelectorAll("[data-cs-panel]").length,
|
||
active: root.querySelector("[data-cs-panel]:not([hidden])")?.dataset.csPanel,
|
||
sourceCards: root.querySelectorAll("[data-cs-source-cards] > article").length,
|
||
seedDots: root.querySelectorAll(".seed-dots > i").length,
|
||
outputCount: root.querySelector("[data-cs-output-count]").textContent.trim(),
|
||
naturalEos: root.querySelector("[data-cs-natural-eos]").textContent.trim(),
|
||
facts,
|
||
labs: [...document.querySelectorAll(".page-facts > div")]
|
||
.find((node) => node.querySelector("dt")?.textContent.trim() === "LABS")
|
||
?.querySelector("dd")?.textContent.trim(),
|
||
status: [...document.querySelectorAll(".page-facts > div")]
|
||
.find((node) => node.querySelector("dt")?.textContent.trim() === "STATUS")
|
||
?.querySelector("dd")?.textContent.trim(),
|
||
overflow: document.documentElement.scrollWidth
|
||
- document.documentElement.clientWidth,
|
||
};
|
||
})()`);
|
||
await pause(250);
|
||
await screenshot("/tmp/llm-atlas-cross-source-desktop.png");
|
||
|
||
const hierarchySwitch = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
const domain = root.querySelector("[data-cs-domain]");
|
||
const condition = root.querySelector("[data-cs-condition]");
|
||
domain.value = "code";
|
||
domain.dispatchEvent(new Event("change", { bubbles: true }));
|
||
condition.value = "s1_period";
|
||
condition.dispatchEvent(new Event("change", { bubbles: true }));
|
||
return {
|
||
cards: [...root.querySelectorAll("[data-cs-source-cards] > article")].map((node) => ({
|
||
id: node.dataset.sourceId,
|
||
eos: node.querySelector("header em").textContent.trim(),
|
||
footer: node.querySelector(":scope > p").textContent.trim(),
|
||
dots: node.querySelectorAll(".seed-dots > i").length,
|
||
})),
|
||
outputCount: root.querySelector("[data-cs-output-count]").textContent.trim(),
|
||
naturalEos: root.querySelector("[data-cs-natural-eos]").textContent.trim(),
|
||
};
|
||
})()`);
|
||
|
||
const taskMatrices = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
root.querySelector('[data-cs-tab="tasks"]').click();
|
||
const read = () => ({
|
||
active: root.querySelector("[data-cs-panel]:not([hidden])").dataset.csPanel,
|
||
total: root.querySelector("[data-cs-task-total]").textContent.trim(),
|
||
rows: [...root.querySelectorAll("[data-cs-task-matrix] > div")].map((row) => ({
|
||
id: row.dataset.taskId,
|
||
cells: [...row.querySelectorAll("b")].map((cell) => cell.textContent.trim()),
|
||
total: row.querySelector("strong").textContent.trim(),
|
||
})),
|
||
range: root.querySelector("[data-cs-task-range]").textContent.trim(),
|
||
interaction: root.querySelector("[data-cs-task-interaction]").textContent.trim(),
|
||
failure: root.querySelector("[data-cs-task-failure]").textContent.trim(),
|
||
});
|
||
const math = read();
|
||
root.querySelector('[data-cs-task-domain="code"]').click();
|
||
const code = read();
|
||
return { math, code };
|
||
})()`);
|
||
await pause(150);
|
||
await screenshot("/tmp/llm-atlas-cross-source-task-matrix.png");
|
||
|
||
const directions = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
root.querySelector('[data-cs-tab="directions"]').click();
|
||
const select = root.querySelector("[data-cs-direction-domain]");
|
||
const read = () => ({
|
||
active: root.querySelector("[data-cs-panel]:not([hidden])").dataset.csPanel,
|
||
shorter: root.querySelector("[data-cs-shorter]").textContent.trim(),
|
||
mean: root.querySelector("[data-cs-domain-mean]").textContent.trim(),
|
||
median: root.querySelector("[data-cs-domain-median]").textContent.trim(),
|
||
rows: [...root.querySelectorAll("[data-cs-direction-rows] > article")].map((row) => ({
|
||
id: row.dataset.directionSource,
|
||
value: row.querySelector("b").textContent.trim(),
|
||
dotClass: row.querySelector("u").className,
|
||
})),
|
||
});
|
||
select.value = "english";
|
||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||
const english = read();
|
||
select.value = "code";
|
||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||
const code = read();
|
||
return { english, code };
|
||
})()`);
|
||
|
||
const keyboard = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
const first = root.querySelector('[data-cs-tab="hierarchy"]');
|
||
first.click();
|
||
first.focus();
|
||
first.dispatchEvent(new KeyboardEvent("keydown", {
|
||
key: "ArrowRight", bubbles: true,
|
||
}));
|
||
return {
|
||
selected: root.querySelector('[data-cs-tab][aria-selected="true"]').dataset.csTab,
|
||
active: root.querySelector("[data-cs-panel]:not([hidden])").dataset.csPanel,
|
||
focused: document.activeElement.dataset.csTab,
|
||
};
|
||
})()`);
|
||
|
||
const reproduction = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
root.querySelector('[data-cs-tab="reproduction"]').click();
|
||
return {
|
||
fields: [...root.querySelectorAll(".repro-fields article")].map((node) => (
|
||
node.querySelector("b").textContent.trim()
|
||
)),
|
||
artifacts: [...root.querySelectorAll(".hash-chain article")].map((node) => ({
|
||
label: node.querySelector("span").textContent.trim(),
|
||
hash: node.querySelector("b").textContent.trim(),
|
||
})),
|
||
active: root.querySelector("[data-cs-panel]:not([hidden])").dataset.csPanel,
|
||
};
|
||
})()`);
|
||
|
||
await command("Emulation.setDeviceMetricsOverride", {
|
||
width: 390,
|
||
height: 844,
|
||
deviceScaleFactor: 1,
|
||
mobile: true,
|
||
});
|
||
await pause(300);
|
||
const mobile = await evaluate(`(() => {
|
||
const root = document.querySelector("[data-cross-source-lab]");
|
||
root.querySelector('[data-cs-tab="tasks"]').click();
|
||
root.scrollIntoView();
|
||
return {
|
||
documentOverflow: document.documentElement.scrollWidth
|
||
- document.documentElement.clientWidth,
|
||
rootOverflow: root.scrollWidth - root.clientWidth,
|
||
matrixWidth: root.querySelector("[data-cs-task-matrix]").getBoundingClientRect().width,
|
||
viewport: window.innerWidth,
|
||
};
|
||
})()`);
|
||
await screenshot("/tmp/llm-atlas-cross-source-mobile.png");
|
||
|
||
assert(overview, "找不到跨来源采样实验");
|
||
assert(overview.tabs === 4 && overview.panels === 4, "四页签/面板合同异常");
|
||
assert(overview.active === "hierarchy", "初始面板不是 hierarchy");
|
||
assert(overview.sourceCards === 4 && overview.seedDots === 16, "source/seed 层级渲染异常");
|
||
assert(overview.outputCount === "16 / 16", "初始输出总账异常");
|
||
assert(overview.facts.SOURCES === "16", "source headline 异常");
|
||
assert(overview.facts["SAMPLED OUTPUTS"] === "256", "output headline 异常");
|
||
assert(overview.facts["NATURAL EOS"] === "250 / 256", "EOS headline 异常");
|
||
assert(overview.facts["MATH · STRICT"] === "47 / 64", "Math headline 异常");
|
||
assert(overview.facts["CODE · TESTS"] === "52 / 64", "Code headline 异常");
|
||
assert(overview.labs === "21 个可操作实验", "DeepSeek LABS 总账异常");
|
||
assert(overview.status === "七轮 · 512 条采样", "DeepSeek STATUS 总账异常");
|
||
assert(overview.overflow <= 1, `桌面横向溢出 ${overview.overflow}px`);
|
||
|
||
assert(hierarchySwitch.cards.length === 4, "Code source cards 数量异常");
|
||
assert(hierarchySwitch.cards.every((row) => row.dots === 4), "每题 seed 数不为 4");
|
||
assert(
|
||
hierarchySwitch.cards.find((row) => row.id === "HumanEval/133")
|
||
?.footer.includes("0 / 4 strict task pass"),
|
||
"HumanEval/133 × s1_period 应为 0/4",
|
||
);
|
||
assert(hierarchySwitch.naturalEos === "16 / 16", "Code s1_period EOS 异常");
|
||
|
||
assert(taskMatrices.math.total === "47 / 64", "Math total 异常");
|
||
assert(
|
||
taskMatrices.math.rows.map((row) => row.cells.join(",")).join("|")
|
||
=== "3 / 4,4 / 4,3 / 4,4 / 4|4 / 4,4 / 4,4 / 4,4 / 4|2 / 4,2 / 4,3 / 4,1 / 4|1 / 4,4 / 4,1 / 4,3 / 4",
|
||
"Math 4×4 pass matrix 异常",
|
||
);
|
||
assert(taskMatrices.math.range === "8 → 16 / 16", "Math source range 异常");
|
||
assert(taskMatrices.math.failure === "17 / 64", "Math failure 账异常");
|
||
assert(taskMatrices.code.total === "52 / 64", "Code total 异常");
|
||
assert(
|
||
taskMatrices.code.rows.map((row) => row.cells.join(",")).join("|")
|
||
=== "4 / 4,4 / 4,4 / 4,4 / 4|2 / 4,1 / 4,1 / 4,4 / 4|4 / 4,4 / 4,4 / 4,0 / 4|4 / 4,4 / 4,4 / 4,4 / 4",
|
||
"Code 4×4 pass matrix 异常",
|
||
);
|
||
assert(taskMatrices.code.interaction === "1↑ · 2＝ · 1↓", "Code interaction 方向异常");
|
||
assert(taskMatrices.code.failure === "12 / 64", "Code failure 账异常");
|
||
|
||
assert(directions.english.shorter === "1 / 4", "English 方向数异常");
|
||
assert(directions.english.mean === "−8.5 tokens".replace("−", "-"), "English mean 异常");
|
||
assert(directions.english.median === "+21.2 tokens", "English median 异常");
|
||
assert(
|
||
directions.english.rows.map((row) => row.value).join("|")
|
||
=== "-121.0 tokens|+44.5 tokens|+41.0 tokens|+1.4 tokens",
|
||
"English source contrast 异常",
|
||
);
|
||
assert(directions.code.shorter === "4 / 4", "Code 方向数异常");
|
||
assert(directions.code.rows.every((row) => row.dotClass === "negative"), "Code 应四条全负");
|
||
|
||
assert(
|
||
keyboard.selected === "tasks"
|
||
&& keyboard.active === "tasks"
|
||
&& keyboard.focused === "tasks",
|
||
"页签键盘导航异常",
|
||
);
|
||
assert(reproduction.active === "reproduction", "复现面板切换异常");
|
||
assert(reproduction.fields.length === 8, "复现字段数不为 8");
|
||
assert(reproduction.fields.every((value) => value === "64 / 64"), "复现字段未全部 exact");
|
||
assert(reproduction.artifacts.length === 5, "hash chain 工件数不为 5");
|
||
assert(mobile.documentOverflow <= 1, `移动端 document 横向溢出 ${mobile.documentOverflow}px`);
|
||
assert(mobile.rootOverflow <= 1, `移动端实验横向溢出 ${mobile.rootOverflow}px`);
|
||
assert(mobile.matrixWidth <= mobile.viewport, "移动端任务矩阵超出 viewport");
|
||
assert(exceptions.length === 0, `浏览器异常：${exceptions.join(" | ")}`);
|
||
|
||
console.log(JSON.stringify({
|
||
overview,
|
||
hierarchySwitch,
|
||
taskMatrices,
|
||
directions,
|
||
keyboard,
|
||
reproduction,
|
||
mobile,
|
||
screenshots: [
|
||
"/tmp/llm-atlas-cross-source-desktop.png",
|
||
"/tmp/llm-atlas-cross-source-task-matrix.png",
|
||
"/tmp/llm-atlas-cross-source-mobile.png",
|
||
],
|
||
}, null, 2));
|
||
|
||
socket.close();
|