wuyang
294 lines
15 KiB
JavaScript
294 lines
15 KiB
JavaScript
import { writeFileSync } from "node:fs";
|
|
|
|
const cdpPort = process.env.CDP_PORT ?? "9227";
|
|
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
|
|
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
|
|
const page = pages.find((entry) => entry.type === "page");
|
|
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
|
|
|
|
const socket = new WebSocket(page.webSocketDebuggerUrl);
|
|
await new Promise((resolve, reject) => {
|
|
socket.addEventListener("open", resolve, { once: true });
|
|
socket.addEventListener("error", reject, { once: true });
|
|
});
|
|
|
|
let nextId = 0;
|
|
const pending = new Map();
|
|
const exceptions = [];
|
|
socket.addEventListener("message", (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
const { resolve, reject } = pending.get(message.id);
|
|
pending.delete(message.id);
|
|
if (message.error) reject(new Error(message.error.message));
|
|
else resolve(message.result);
|
|
}
|
|
if (message.method === "Runtime.exceptionThrown") {
|
|
exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
|
|
}
|
|
});
|
|
|
|
const command = (method, params = {}) => new Promise((resolve, reject) => {
|
|
const id = ++nextId;
|
|
pending.set(id, { resolve, reject });
|
|
socket.send(JSON.stringify({ id, method, params }));
|
|
});
|
|
const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
const evaluate = async (expression) => {
|
|
const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
|
|
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
|
|
return result.result.value;
|
|
};
|
|
const navigate = async (path) => {
|
|
await command("Page.navigate", { url: `${baseUrl}${path}` });
|
|
for (let attempt = 0; attempt < 70; attempt += 1) {
|
|
await pause(100);
|
|
if (await evaluate("document.readyState === 'complete'")) return;
|
|
}
|
|
throw new Error(`${path} 加载超时`);
|
|
};
|
|
const screenshot = async (path) => {
|
|
const result = await command("Page.captureScreenshot", { format: "png", captureBeyondViewport: false });
|
|
writeFileSync(path, Buffer.from(result.data, "base64"));
|
|
};
|
|
|
|
await command("Page.enable");
|
|
await command("Runtime.enable");
|
|
await command("Emulation.setDeviceMetricsOverride", {
|
|
width: 1440,
|
|
height: 1100,
|
|
deviceScaleFactor: 1,
|
|
mobile: false,
|
|
});
|
|
|
|
await navigate("/evaluation/");
|
|
await screenshot("/tmp/llm-atlas-evaluation-desktop.png");
|
|
|
|
const overview = await evaluate(`(() => ({
|
|
title: document.querySelector("h1")?.textContent.trim(),
|
|
sections: document.querySelectorAll(".article-section").length,
|
|
tocLinks: document.querySelectorAll(".side-rail a").length,
|
|
paperLinks: document.querySelectorAll(".paper-chain a").length,
|
|
ledgers: document.querySelectorAll(".eval-ledgers > article").length,
|
|
labTabs: document.querySelectorAll("[data-eval-tab]").length,
|
|
labPanels: document.querySelectorAll("[data-eval-panel]").length,
|
|
navLinks: document.querySelectorAll(".top-nav a").length,
|
|
activeNav: document.querySelector('.top-nav a[aria-current="page"]')?.textContent.trim(),
|
|
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
|
}))()`);
|
|
|
|
const metric = await evaluate(`(() => {
|
|
const root = document.querySelector("[data-evaluation-lab]");
|
|
const read = () => ({
|
|
one: root.querySelector("[data-one-try]").textContent.trim(),
|
|
atLeast: root.querySelector("[data-at-least]").textContent.trim(),
|
|
all: root.querySelector("[data-all-succeed]").textContent.trim(),
|
|
ece: root.querySelector("[data-toy-ece]").textContent.trim(),
|
|
bpb: root.querySelector("[data-bpb]").textContent.trim(),
|
|
pplA: root.querySelector("[data-ppl-a]").textContent.trim(),
|
|
pplB: root.querySelector("[data-ppl-b]").textContent.trim(),
|
|
visible: root.querySelector("[data-eval-panel]:not([hidden])").dataset.evalPanel,
|
|
explain: root.querySelector("[data-metric-explain]").textContent.trim(),
|
|
});
|
|
const initial = read();
|
|
root.querySelector('[data-protocol-preset="search"]').click();
|
|
const search = read();
|
|
root.querySelector('[data-protocol-preset="reliable"]').click();
|
|
const reliable = read();
|
|
return { initial, search, reliable };
|
|
})()`);
|
|
|
|
const judge = await evaluate(`(() => {
|
|
const root = document.querySelector("[data-evaluation-lab]");
|
|
root.querySelector('[data-eval-tab="judge"]').click();
|
|
const read = () => ({
|
|
raw: root.querySelector("[data-raw-win]").textContent.trim(),
|
|
adjusted: root.querySelector("[data-adjusted-win]").textContent.trim(),
|
|
flip: root.querySelector("[data-order-flip]").textContent.trim(),
|
|
ci: root.querySelector("[data-judge-ci]").textContent.trim(),
|
|
explain: root.querySelector("[data-judge-explain]").textContent.trim(),
|
|
});
|
|
const initial = read();
|
|
const gap = root.querySelector("[data-length-gap]");
|
|
const bias = root.querySelector("[data-length-bias]");
|
|
const order = root.querySelector("[data-order]");
|
|
const votes = root.querySelector("[data-votes]");
|
|
gap.value = "100";
|
|
gap.dispatchEvent(new Event("input", { bubbles: true }));
|
|
bias.value = "40";
|
|
bias.dispatchEvent(new Event("input", { bubbles: true }));
|
|
const biased = read();
|
|
order.value = "randomized";
|
|
order.dispatchEvent(new Event("input", { bubbles: true }));
|
|
votes.value = "10000";
|
|
votes.dispatchEvent(new Event("input", { bubbles: true }));
|
|
const controlled = read();
|
|
return { initial, biased, controlled };
|
|
})()`);
|
|
|
|
const contamination = await evaluate(`(() => {
|
|
const root = document.querySelector("[data-evaluation-lab]");
|
|
root.querySelector('[data-eval-tab="contamination"]').click();
|
|
const read = () => ({
|
|
clean: root.querySelector("[data-true-clean]").textContent.trim(),
|
|
detected: root.querySelector("[data-detected]").textContent.trim(),
|
|
hidden: root.querySelector("[data-hidden-leak]").textContent.trim(),
|
|
fresh: root.querySelector("[data-freshness]").textContent.trim(),
|
|
comparable: root.querySelector("[data-comparability]").textContent.trim(),
|
|
explain: root.querySelector("[data-contamination-explain]").textContent.trim(),
|
|
});
|
|
const initial = read();
|
|
root.querySelectorAll("[data-leak]").forEach((input) => {
|
|
input.checked = input.dataset.leak === "semantic";
|
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
|
});
|
|
const ngram = root.querySelector("[data-ngram]");
|
|
ngram.value = "30";
|
|
ngram.dispatchEvent(new Event("input", { bubbles: true }));
|
|
const semantic = read();
|
|
const refresh = root.querySelector("[data-refresh]");
|
|
const anchor = root.querySelector("[data-anchor]");
|
|
refresh.value = "90";
|
|
refresh.dispatchEvent(new Event("input", { bubbles: true }));
|
|
anchor.value = "10";
|
|
anchor.dispatchEvent(new Event("input", { bubbles: true }));
|
|
const dynamic = read();
|
|
return { initial, semantic, dynamic };
|
|
})()`);
|
|
|
|
const system = await evaluate(`(() => {
|
|
const root = document.querySelector("[data-evaluation-lab]");
|
|
root.querySelector('[data-eval-tab="system"]').click();
|
|
const read = () => ({
|
|
model: root.querySelector("[data-model-only]").textContent.trim(),
|
|
success: root.querySelector("[data-system-success]").textContent.trim(),
|
|
cost: root.querySelector("[data-cost]").textContent.trim(),
|
|
unsafe: root.querySelector("[data-unsafe]").textContent.trim(),
|
|
overrefusal: root.querySelector("[data-overrefusal]").textContent.trim(),
|
|
explain: root.querySelector("[data-system-explain]").textContent.trim(),
|
|
});
|
|
const initial = read();
|
|
const retries = root.querySelector("[data-retries]");
|
|
const budget = root.querySelector("[data-budget]");
|
|
retries.value = "1";
|
|
retries.dispatchEvent(new Event("input", { bubbles: true }));
|
|
budget.value = "4";
|
|
budget.dispatchEvent(new Event("input", { bubbles: true }));
|
|
const cheap = read();
|
|
const wrapper = root.querySelector("[data-wrapper]");
|
|
wrapper.value = "100";
|
|
wrapper.dispatchEvent(new Event("input", { bubbles: true }));
|
|
const locked = read();
|
|
const firstTab = root.querySelector('[data-eval-tab="metric"]');
|
|
firstTab.focus();
|
|
firstTab.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
|
return {
|
|
initial,
|
|
cheap,
|
|
locked,
|
|
keyboardSelected: root.querySelector('[data-eval-tab][aria-selected="true"]').dataset.evalTab,
|
|
keyboardVisible: root.querySelector("[data-eval-panel]:not([hidden])").dataset.evalPanel,
|
|
};
|
|
})()`);
|
|
|
|
await evaluate(`document.querySelector("[data-evaluation-lab]").scrollIntoView({ block: "start", behavior: "instant" })`);
|
|
await pause(180);
|
|
await screenshot("/tmp/llm-atlas-evaluation-lab-desktop.png");
|
|
|
|
await navigate("/");
|
|
const home = await evaluate(`(() => ({
|
|
releaseCards: document.querySelectorAll(".release-card").length,
|
|
firstRelease: document.querySelector(".release-card h2").textContent.trim(),
|
|
firstHref: document.querySelector(".release-card").getAttribute("href"),
|
|
paperCount: document.querySelector(".hero-stats div:nth-child(3) b").textContent.trim(),
|
|
topicCount: document.querySelector(".hero-stats div:nth-child(1) b").textContent.trim(),
|
|
navLinks: document.querySelectorAll(".top-nav a").length,
|
|
}))()`);
|
|
|
|
await navigate("/papers/");
|
|
const papers = await evaluate(`(() => {
|
|
const button = [...document.querySelectorAll("[data-filter]")].find((node) => node.textContent.trim() === "评测");
|
|
button?.click();
|
|
return {
|
|
total: document.querySelectorAll("[data-paper]").length,
|
|
visible: document.querySelectorAll("[data-paper]:not([hidden])").length,
|
|
hasFilter: Boolean(button),
|
|
};
|
|
})()`);
|
|
|
|
await command("Emulation.setDeviceMetricsOverride", {
|
|
width: 390,
|
|
height: 844,
|
|
deviceScaleFactor: 1,
|
|
mobile: true,
|
|
});
|
|
await navigate("/evaluation/");
|
|
const mobile = await evaluate(`(() => {
|
|
const root = document.querySelector("[data-evaluation-lab]");
|
|
root.scrollIntoView({ block: "start", behavior: "instant" });
|
|
const toggle = document.querySelector("#menu-toggle");
|
|
toggle?.click();
|
|
return {
|
|
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
|
menuVisible: getComputedStyle(toggle).display !== "none",
|
|
menuOpen: toggle.getAttribute("aria-expanded"),
|
|
mobileLinks: document.querySelectorAll("#mobile-nav a").length,
|
|
tabs: root.querySelectorAll("[data-eval-tab]").length,
|
|
offenders: [...document.querySelectorAll("body *")]
|
|
.filter((node) => !node.closest(".aggregate-example, .repair-table, .k3-protocol-table, .paper-chain"))
|
|
.filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
|
|
.slice(0, 12)
|
|
.map((node) => ({
|
|
tag: node.tagName,
|
|
className: typeof node.className === "string" ? node.className : "",
|
|
right: Math.round(node.getBoundingClientRect().right),
|
|
width: Math.round(node.getBoundingClientRect().width),
|
|
})),
|
|
};
|
|
})()`);
|
|
await pause(180);
|
|
await screenshot("/tmp/llm-atlas-evaluation-mobile.png");
|
|
|
|
const report = { overview, metric, judge, contamination, system, home, papers, mobile, exceptions };
|
|
console.log(JSON.stringify(report, null, 2));
|
|
|
|
const numeric = (text) => Number.parseFloat(text.replaceAll(",", ""));
|
|
const intervalWidth = (text) => {
|
|
const values = text.match(/[0-9.]+/g)?.map(Number) ?? [];
|
|
return values.length >= 2 ? values[1] - values[0] : Number.NaN;
|
|
};
|
|
const failures = [];
|
|
if (!overview.title.includes("粗体数字")) failures.push("章节标题异常");
|
|
if (overview.sections !== 33 || overview.tocLinks !== 33) failures.push("章节 / 目录数量异常");
|
|
if (overview.paperLinks !== 80) failures.push("正式论文链不是 80 个节点");
|
|
if (overview.ledgers !== 22) failures.push("二十二张评测账结构异常");
|
|
if (overview.labTabs !== 4 || overview.labPanels !== 4) failures.push("四联实验结构异常");
|
|
if (overview.navLinks !== 20 || home.navLinks !== 20 || mobile.mobileLinks !== 20 || overview.activeNav !== "评测安全") failures.push("全站导航未同步评测专题");
|
|
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在文档级横向溢出");
|
|
if (metric.initial.visible !== "metric" || numeric(metric.initial.one) !== 80 || numeric(metric.initial.atLeast) !== 80 || numeric(metric.initial.all) !== 80) failures.push("指标实验初始值异常");
|
|
if (numeric(metric.search.atLeast) <= 99 || numeric(metric.search.all) >= 30 || !metric.search.explain.includes("搜索")) failures.push("pass@k / pass^k 方向没有分开");
|
|
if (Math.abs(numeric(metric.reliable.all) - 32.8) > .2) failures.push("连续五次可靠性计算异常");
|
|
if (numeric(metric.initial.pplB) <= numeric(metric.initial.pplA) || !metric.initial.bpb.includes("bits")) failures.push("tokenizer / BPB 教学对照异常");
|
|
if (numeric(judge.biased.raw) >= numeric(judge.biased.adjusted)) failures.push("长度偏好没有压低短回答的观察胜率");
|
|
if (!judge.controlled.explain.includes("随机交换") || intervalWidth(judge.controlled.ci) >= intervalWidth(judge.initial.ci)) failures.push("顺序控制或票数区间异常");
|
|
if (numeric(contamination.semantic.detected) >= 10 || numeric(contamination.semantic.hidden) <= 15 || !contamination.semantic.explain.includes("语义")) failures.push("语义污染没有暴露 n-gram 检测盲区");
|
|
if (numeric(contamination.dynamic.fresh) <= numeric(contamination.initial.fresh) || numeric(contamination.dynamic.comparable) >= numeric(contamination.initial.comparable)) failures.push("动态刷新与可比性权衡异常");
|
|
if (numeric(system.initial.success) <= numeric(system.initial.model) || numeric(system.initial.cost) !== 128) failures.push("Harness / 重试系统成功率或成本异常");
|
|
if (numeric(system.cheap.success) >= numeric(system.initial.success) || numeric(system.cheap.cost) !== 4) failures.push("低预算没有降低成功率 / 成本");
|
|
if (numeric(system.locked.unsafe) !== 0 || numeric(system.locked.overrefusal) <= numeric(system.initial.overrefusal)) failures.push("安全壳没有展现危险服从 / 过拒权衡");
|
|
if (system.keyboardSelected !== "judge" || system.keyboardVisible !== "judge") failures.push("实验键盘 tab 导航异常");
|
|
if (home.releaseCards !== 17 || !home.firstRelease.includes("局部 uniform routing 进入完整训练后") || home.firstHref !== "/k3/") failures.push("首页 K3 首发入口异常");
|
|
if (home.paperCount !== "486" || home.topicCount !== "17" || papers.total !== 486 || !papers.hasFilter || papers.visible < 80) failures.push("首页 / 论文库评测索引异常");
|
|
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4) failures.push("移动端导航或实验异常");
|
|
if (mobile.offenders.length) failures.push(`移动端越界元素：${JSON.stringify(mobile.offenders)}`);
|
|
if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);
|
|
|
|
if (failures.length) {
|
|
console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
|
|
process.exitCode = 1;
|
|
} else {
|
|
console.log("\nPASS evaluation browser regression");
|
|
}
|
|
|
|
socket.close();
|