feat: publish long-horizon agent chapter

This commit is contained in:
wuyang
2026-07-29 06:54:31 +08:00
parent 6c8c0fb90f
commit 25982cbbb4
25 changed files with 5122 additions and 39 deletions
+243
View File
@@ -0,0 +1,243 @@
import { writeFileSync } from "node:fs";
const cdpPort = process.env.CDP_PORT ?? "9225";
const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4323";
const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
const page = pages.find((entry) => entry.type === "page");
if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
const socket = new WebSocket(page.webSocketDebuggerUrl);
await new Promise((resolve, reject) => {
socket.addEventListener("open", resolve, { once: true });
socket.addEventListener("error", reject, { once: true });
});
let nextId = 0;
const pending = new Map();
const exceptions = [];
socket.addEventListener("message", (event) => {
const message = JSON.parse(event.data);
if (message.id && pending.has(message.id)) {
const { resolve, reject } = pending.get(message.id);
pending.delete(message.id);
if (message.error) reject(new Error(message.error.message));
else resolve(message.result);
}
if (message.method === "Runtime.exceptionThrown") {
exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
}
});
const command = (method, params = {}) => new Promise((resolve, reject) => {
const id = ++nextId;
pending.set(id, { resolve, reject });
socket.send(JSON.stringify({ id, method, params }));
});
const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
const evaluate = async (expression) => {
const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
return result.result.value;
};
const navigate = async (path) => {
await command("Page.navigate", { url: `${baseUrl}${path}` });
for (let attempt = 0; attempt < 70; attempt += 1) {
await pause(100);
if (await evaluate("document.readyState === 'complete'")) return;
}
throw new Error(`${path} 加载超时`);
};
const screenshot = async (path) => {
const result = await command("Page.captureScreenshot", {
format: "png",
captureBeyondViewport: false,
});
writeFileSync(path, Buffer.from(result.data, "base64"));
};
await command("Page.enable");
await command("Runtime.enable");
await command("Emulation.setDeviceMetricsOverride", {
width: 1440,
height: 1100,
deviceScaleFactor: 1,
mobile: false,
});
await navigate("/agents/");
await screenshot("/tmp/llm-atlas-agents-desktop.png");
const overview = await evaluate(`(() => ({
title: document.querySelector("h1")?.textContent.trim(),
sections: document.querySelectorAll(".article-section").length,
tocLinks: document.querySelectorAll(".side-rail a").length,
paperLinks: document.querySelectorAll(".paper-chain a").length,
labTabs: document.querySelectorAll("[data-agent-tab]").length,
labPanels: document.querySelectorAll("[data-agent-panel]").length,
navLinks: document.querySelectorAll(".top-nav a").length,
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
}))()`);
const loop = await evaluate(`(() => {
const root = document.querySelector("[data-agent-lab]");
const read = () => ({
status: root.querySelector("[data-loop-status]").textContent.trim(),
observation: root.querySelector("[data-loop-observation]").textContent.trim(),
recovery: root.querySelector("[data-loop-recovery]").textContent.trim(),
finalState: root.querySelector("[data-loop-final]").textContent.trim(),
takeaway: root.querySelector("[data-loop-takeaway]").textContent.trim(),
});
const initial = read();
root.querySelector('[data-loop-mode="direct"]').click();
root.querySelector('[data-loop-fault="schema"]').click();
root.querySelector("[data-loop-next]").click();
root.querySelector("[data-loop-next]").click();
return { initial, directSchema: read() };
})()`);
const contract = await evaluate(`(() => {
const root = document.querySelector("[data-agent-lab]");
root.querySelector('[data-agent-tab="contract"]').click();
root.querySelector('[data-contract-case="wrongstate"]').click();
return {
syntax: root.querySelector('[data-contract-score="syntax"]').textContent.trim(),
execution: root.querySelector('[data-contract-score="execution"]').textContent.trim(),
finalState: root.querySelector('[data-contract-score="final"]').textContent.trim(),
takeaway: root.querySelector("[data-contract-takeaway]").textContent.trim(),
visiblePanel: root.querySelector("[data-agent-panel]:not([hidden])").dataset.agentPanel,
};
})()`);
const reliability = await evaluate(`(() => {
const root = document.querySelector("[data-agent-lab]");
root.querySelector('[data-agent-tab="reliability"]').click();
const k = root.querySelector("[data-rel-k]");
const read = () => ({
passAt: root.querySelector("[data-rel-pass-at]").textContent.trim(),
passHat: root.querySelector("[data-rel-pass-hat]").textContent.trim(),
sideRisk: root.querySelector("[data-rel-side]").textContent.trim(),
takeaway: root.querySelector("[data-rel-takeaway]").textContent.trim(),
});
const initial = read();
k.value = "2";
k.dispatchEvent(new Event("input", { bubbles: true }));
const k2 = read();
k.value = "8";
k.dispatchEvent(new Event("input", { bubbles: true }));
root.querySelector('[data-rel-idempotent="false"]').click();
const nonIdempotent = read();
return { initial, k2, nonIdempotent };
})()`);
const rl = await evaluate(`(() => {
const root = document.querySelector("[data-agent-lab]");
root.querySelector('[data-agent-tab="rl"]').click();
const read = () => ({
utilization: root.querySelector("[data-rl-util]").textContent.trim(),
lostWork: root.querySelector("[data-rl-lost]").textContent.trim(),
modelState: root.querySelector("[data-rl-model-state]").textContent.trim(),
worldState: root.querySelector("[data-rl-world-state]").textContent.trim(),
takeaway: root.querySelector("[data-rl-takeaway]").textContent.trim(),
});
const full = read();
root.querySelector('[data-rl-mode="wait"]').click();
const wait = read();
const firstTab = root.querySelector('[data-agent-tab="loop"]');
firstTab.focus();
firstTab.dispatchEvent(new KeyboardEvent("keydown", { key: "End", bubbles: true }));
return {
full,
wait,
keyboardSelected: root.querySelector('[data-agent-tab][aria-selected="true"]').dataset.agentTab,
keyboardVisible: root.querySelector("[data-agent-panel]:not([hidden])").dataset.agentPanel,
};
})()`);
await evaluate(`document.querySelector("[data-agent-lab]").scrollIntoView({ block: "start", behavior: "instant" })`);
await pause(180);
await screenshot("/tmp/llm-atlas-agents-lab-desktop.png");
await navigate("/");
const home = await evaluate(`(() => ({
releaseCards: document.querySelectorAll(".release-card").length,
firstRelease: document.querySelector(".release-card h2").textContent.trim(),
firstHref: document.querySelector(".release-card").getAttribute("href"),
paperCount: document.querySelector(".hero-stats div:nth-child(3) b").textContent.trim(),
navLinks: document.querySelectorAll(".top-nav a").length,
}))()`);
await navigate("/papers/");
const papers = await evaluate(`(() => {
const button = [...document.querySelectorAll("[data-filter]")].find((node) => node.textContent.trim() === "Agent");
button?.click();
return {
total: document.querySelectorAll("[data-paper]").length,
agentVisible: document.querySelectorAll("[data-paper]:not([hidden])").length,
hasAgentFilter: Boolean(button),
};
})()`);
await command("Emulation.setDeviceMetricsOverride", {
width: 390,
height: 844,
deviceScaleFactor: 1,
mobile: true,
});
await navigate("/agents/");
const mobile = await evaluate(`(() => {
const root = document.querySelector("[data-agent-lab]");
root.scrollIntoView({ block: "start", behavior: "instant" });
const toggle = document.querySelector("#menu-toggle");
toggle?.click();
return {
documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
menuVisible: getComputedStyle(toggle).display !== "none",
menuOpen: toggle.getAttribute("aria-expanded"),
mobileLinks: document.querySelectorAll("#mobile-nav a").length,
tabs: root.querySelectorAll("[data-agent-tab]").length,
offenders: [...document.querySelectorAll("body *")]
.filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
.slice(0, 12)
.map((node) => ({
tag: node.tagName,
className: typeof node.className === "string" ? node.className : "",
right: Math.round(node.getBoundingClientRect().right),
width: Math.round(node.getBoundingClientRect().width),
})),
};
})()`);
await pause(180);
await screenshot("/tmp/llm-atlas-agents-mobile.png");
const report = { overview, loop, contract, reliability, rl, home, papers, mobile, exceptions };
console.log(JSON.stringify(report, null, 2));
const numeric = (value) => Number.parseFloat(value);
const failures = [];
if (!overview.title.includes("可靠行动")) failures.push("章节标题异常");
if (overview.sections !== 28 || overview.tocLinks !== 28) failures.push("章节/目录数量异常");
if (overview.paperLinks !== 52) failures.push("正式论文链不是 52 个节点");
if (overview.labTabs !== 4 || overview.labPanels !== 4) failures.push("四联实验结构异常");
if (overview.navLinks !== 16 || home.navLinks !== 16 || mobile.mobileLinks !== 16) failures.push("全站导航未同步 Agent 专题");
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在横向溢出");
if (loop.initial.finalState !== "UNVERIFIED" || loop.directSchema.finalState !== "FAILED") failures.push("控制循环终局状态异常");
if (!loop.directSchema.observation.includes("ERROR schema") || loop.directSchema.recovery !== "FRAGILE") failures.push("Direct/schema 故障传播异常");
if (contract.syntax !== "100%" || contract.execution !== "100%" || contract.finalState !== "0%") failures.push("工具契约三层评分异常");
if (!contract.takeaway.includes("final-state") || contract.visiblePanel !== "contract") failures.push("工具契约终局验证解释异常");
if (numeric(reliability.initial.passAt) < 99.9 || Math.abs(numeric(reliability.initial.passHat) - 16.78) > 0.02) failures.push("pass@k / pass^k 初始计算异常");
if (numeric(reliability.initial.passAt) <= numeric(reliability.k2.passAt) || numeric(reliability.initial.passHat) >= numeric(reliability.k2.passHat)) failures.push("k 增长时两类可靠性没有反向变化");
if (reliability.nonIdempotent.sideRisk === "LOW") failures.push("非幂等写操作风险没有提升");
if (numeric(rl.wait.utilization) >= numeric(rl.full.utilization) || numeric(rl.wait.lostWork) <= numeric(rl.full.lostWork)) failures.push("wait-all 长尾/重算方向异常");
if (!rl.wait.takeaway.includes("wait-all") || rl.keyboardSelected !== "rl" || rl.keyboardVisible !== "rl") failures.push("长程 RL 解释或键盘导航异常");
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环") || home.firstHref !== "/agents/") failures.push("首页 Agent 首发入口异常");
if (home.paperCount !== "314" || papers.total !== 314 || !papers.hasAgentFilter || papers.agentVisible < 52) failures.push("论文库 Agent 标签或论文总数异常");
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4) failures.push("移动端导航或实验异常");
if (exceptions.length) failures.push(`浏览器异常:${exceptions.join(" | ")}`);
if (failures.length) {
console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
process.exitCode = 1;
} else {
console.log("\nPASS agents browser regression");
}
socket.close();
+3 -3
View File
@@ -216,7 +216,7 @@ if (!overview.title.includes("真正与人协作")) failures.push("章节标题
if (overview.sections !== 22 || overview.tocLinks !== 22) failures.push("章节/目录数量异常");
if (overview.paperLinks !== 44) failures.push("正式论文链不是 44 个节点");
if (overview.labTabs !== 4 || overview.labPanels !== 4) failures.push("四联实验结构异常");
if (overview.navLinks !== 15 || home.navLinks !== 15 || mobile.mobileLinks !== 15) failures.push("全站导航未同步后训练专题");
if (overview.navLinks !== 16 || home.navLinks !== 16 || mobile.mobileLinks !== 16) failures.push("全站导航未同步后训练专题");
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在横向溢出");
if (sft.initial.active !== "6 / 10" || !sft.initial.lossStates.slice(0, 4).every((value) => value === "MASKED")) failures.push("SFT response-only mask 异常");
if (sft.unsafeAll.active !== "10 / 10" || numeric(sft.unsafeAll.nll) <= numeric(sft.initial.nll) || !sft.unsafeAll.reading.includes("错误")) failures.push("SFT 全序列/坏示范交互异常");
@@ -226,8 +226,8 @@ if (!update.steps[0].includes("Fixed preference")) failures.push("DPO 更新流
if (!recipe.family.includes("Multi-effort") || !recipe.regime.includes("9 RL experts") || !recipe.constraints.includes("verbosity")) failures.push("K3 配方合同异常");
if (!recipe.path.some((step) => step.includes("3 domains × 3 efforts")) || !recipe.path.some((step) => step.includes("MOPD"))) failures.push("K3 配方路径异常");
if (recipe.keyboardSelected !== "recipe" || recipe.keyboardVisible !== "recipe") failures.push("实验 tab 键盘导航异常");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有") || home.firstHref !== "/post-training/alignment/") failures.push("首页 Alignment 首发入口异常");
if (home.paperCount !== "280" || papers.total !== 280 || !papers.hasAlignmentFilter || papers.alignmentVisible < 35) failures.push("论文库后训练标签或论文总数异常");
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环") || home.firstHref !== "/agents/") failures.push("首页 Alignment 首发入口异常");
if (home.paperCount !== "314" || papers.total !== 314 || !papers.hasAlignmentFilter || papers.alignmentVisible < 35) failures.push("论文库后训练标签或论文总数异常");
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4) failures.push("移动端导航或实验异常");
if (exceptions.length) failures.push(`浏览器异常:${exceptions.join(" | ")}`);
+4 -4
View File
@@ -230,15 +230,15 @@ if (transform.keyboard.selected !== "transform" || transform.keyboard.visible !=
if (layout.articleSections !== 18 || layout.paperLinks !== 31 || layout.labTabs !== 4 || layout.views !== 4) {
failures.push("章节、论文或实验数量异常");
}
if (layout.navLinks !== 15 || mobile.mobileLinks !== 15 || home.navLinks !== 15) failures.push("全站导航未同步数值专题");
if (layout.navLinks !== 16 || mobile.mobileLinks !== 16 || home.navLinks !== 16) failures.push("全站导航未同步数值专题");
if (layout.documentOverflow > 0 || mobile.documentOverflow > 0 || home.documentOverflow > 0) failures.push("页面存在横向溢出");
if (layout.navGap < 0) failures.push(`桌面导航碰撞:${layout.navGap}px`);
if (!mobile.menuVisible || mobile.menuOpen !== "true") failures.push("移动端菜单不可用");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有") || home.firstHref !== "/post-training/alignment/") {
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环") || home.firstHref !== "/agents/") {
failures.push("首页 Transformer 新章入口异常");
}
if (home.paperCount !== "280") failures.push(`首页论文总数异常:${home.paperCount}`);
if (!papers.hasDataFilter || papers.total !== 280 || papers.visible < 25) failures.push("论文库数据标签或论文总数异常");
if (home.paperCount !== "314") failures.push(`首页论文总数异常:${home.paperCount}`);
if (!papers.hasDataFilter || papers.total !== 314 || papers.visible < 25) failures.push("论文库数据标签或论文总数异常");
if (exceptions.length) failures.push(`浏览器脚本异常:${exceptions.join("; ")}`);
socket.close();
+1 -1
View File
@@ -177,7 +177,7 @@ if (layout.documentOverflow > 0 || mobile.documentOverflow > 0 || home.documentO
}
if (layout.navGap < 0) failures.push(`桌面导航碰撞:${layout.navGap}px`);
if (!mobile.menuVisible) failures.push("移动端菜单按钮未显示");
if (home.releaseCards !== 10) failures.push(`首页新章卡数量异常:${home.releaseCards}`);
if (home.releaseCards !== 11) failures.push(`首页新章卡数量异常:${home.releaseCards}`);
if (exceptions.length) failures.push(`浏览器脚本异常:${exceptions.join("; ")}`);
socket.close();
+4 -4
View File
@@ -273,15 +273,15 @@ if (stability.keyboard.selected !== "stability" || stability.keyboard.visible !=
if (layout.articleSections !== 19 || layout.paperLinks !== 36 || layout.labTabs !== 4 || layout.views !== 4) {
failures.push("章节、论文或实验数量异常");
}
if (layout.navLinks !== 15 || mobile.mobileLinks !== 15 || home.navLinks !== 15) failures.push("全站导航未同步数值专题");
if (layout.navLinks !== 16 || mobile.mobileLinks !== 16 || home.navLinks !== 16) failures.push("全站导航未同步数值专题");
if (layout.documentOverflow > 0 || mobile.documentOverflow > 0 || home.documentOverflow > 0) failures.push("页面存在横向溢出");
if (layout.navGap < 0) failures.push(`桌面导航碰撞:${layout.navGap}px`);
if (!mobile.menuVisible || mobile.menuOpen !== "true") failures.push("移动端菜单不可用");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有") || home.firstHref !== "/post-training/alignment/") {
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环") || home.firstHref !== "/agents/") {
failures.push("首页 Transformer 新章入口异常");
}
if (home.paperCount !== "280") failures.push(`首页论文总数异常:${home.paperCount}`);
if (!papers.hasOptimizerFilter || papers.total !== 280 || papers.visible < 8) failures.push("论文库优化器标签或论文总数异常");
if (home.paperCount !== "314") failures.push(`首页论文总数异常:${home.paperCount}`);
if (!papers.hasOptimizerFilter || papers.total !== 314 || papers.visible < 8) failures.push("论文库优化器标签或论文总数异常");
if (exceptions.length) failures.push(`浏览器脚本异常:${exceptions.join("; ")}`);
socket.close();
+1 -1
View File
@@ -289,7 +289,7 @@ if (layout.documentOverflow > 0 || mobile.documentOverflow > 0 || home.documentO
}
if (layout.navGap < 0) failures.push(`桌面导航碰撞:${layout.navGap}px`);
if (!mobile.menuVisible || mobile.menuOpen !== "true") failures.push("移动端菜单不可用");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有")) failures.push("首页 Transformer 新章入口异常");
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环")) failures.push("首页 Transformer 新章入口异常");
if (exceptions.length) failures.push(`浏览器脚本异常:${exceptions.join("; ")}`);
socket.close();
+3 -3
View File
@@ -269,14 +269,14 @@ if (emergence.paths.some((length) => length < 500)) failures.push("涌现多指
if (layout.articleSections !== 16 || layout.paperLinks !== 29 || layout.labTabs !== 4 || layout.views !== 4) {
failures.push("章节、论文或实验数量异常");
}
if (layout.navLinks !== 15 || mobile.mobileLinks !== 15 || home.navLinks !== 15) failures.push("全站导航未同步数值专题");
if (layout.navLinks !== 16 || mobile.mobileLinks !== 16 || home.navLinks !== 16) failures.push("全站导航未同步数值专题");
if (layout.documentOverflow > 0 || mobile.documentOverflow > 0 || home.documentOverflow > 0) failures.push("页面存在横向溢出");
if (layout.navGap < 0) failures.push(`桌面导航碰撞:${layout.navGap}px`);
if (!mobile.menuVisible || mobile.menuOpen !== "true") failures.push("移动端菜单不可用");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有") || home.firstHref !== "/post-training/alignment/") {
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环") || home.firstHref !== "/agents/") {
failures.push("首页 Transformer 新章入口异常");
}
if (home.paperCount !== "280") failures.push(`首页论文总数异常:${home.paperCount}`);
if (home.paperCount !== "314") failures.push(`首页论文总数异常:${home.paperCount}`);
if (exceptions.length) failures.push(`浏览器脚本异常:${exceptions.join("; ")}`);
socket.close();
+1 -1
View File
@@ -233,7 +233,7 @@ if (layout.articleSections !== 16 || layout.paperLinks !== 37 || layout.labTabs
if (layout.documentOverflow > 0 || mobile.documentOverflow > 0 || home.documentOverflow > 0) failures.push("页面存在横向溢出");
if (layout.navGap < 0) failures.push(`桌面导航碰撞:${layout.navGap}px`);
if (!mobile.menuVisible || mobile.menuOpen !== "true") failures.push("移动端菜单不可用");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有")) failures.push("首页 Transformer 新章入口异常");
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环")) failures.push("首页 Transformer 新章入口异常");
if (exceptions.length) failures.push(`浏览器脚本异常:${exceptions.join("; ")}`);
socket.close();
+3 -3
View File
@@ -172,7 +172,7 @@ const home = await evaluate(`(() => ({
releaseCards: document.querySelectorAll(".release-card").length,
firstRelease: document.querySelector(".release-card h2").textContent,
firstHref: document.querySelector(".release-card").getAttribute("href"),
paperCount: [...document.querySelectorAll(".hero-stats b")].map((node) => node.textContent.trim()).find((value) => value === "280"),
paperCount: [...document.querySelectorAll(".hero-stats b")].map((node) => node.textContent.trim()).find((value) => value === "314"),
}))()`);
await navigate("/papers/");
@@ -236,8 +236,8 @@ if (block.family.trim() !== "Hybrid MoE" || !block.kv.includes("3 KDA : 1 Gated
if (!block.path.some((step) => step.includes("KDA × 3")) || !block.note.includes("AttnRes")) failures.push("K3 Block 路径异常");
if (block.context.trim() !== "128K" || numeric(block.mha) !== 400 || numeric(block.kda) !== 1) failures.push("KV 成本缩放异常");
if (block.keyboardSelected !== "block" || block.keyboardVisible !== "block") failures.push("实验 tab 键盘导航异常");
if (home.releaseCards !== 10 || !home.firstRelease.includes("DPO 没有") || home.firstHref !== "/post-training/alignment/") failures.push("首页 Transformer 首发入口异常");
if (home.paperCount !== "280" || papers.total !== 280 || papers.transformerVisible < 30) failures.push("论文库或首页论文数量异常");
if (home.releaseCards !== 11 || !home.firstRelease.includes("Agent 不是一个循环") || home.firstHref !== "/agents/") failures.push("首页 Transformer 首发入口异常");
if (home.paperCount !== "314" || papers.total !== 314 || papers.transformerVisible < 30) failures.push("论文库或首页论文数量异常");
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4) failures.push("移动端导航或实验异常");
if (exceptions.length) failures.push(`浏览器异常:${exceptions.join(" | ")}`);