feat: add AttnRes gradient scale lab

2026-07-30 10:24:42 +08:00
parent 26fc824409
commit f7670efcdd
9 changed files with 1191 additions and 17 deletions
@@ -0,0 +1,231 @@
+import { writeFileSync } from "node:fs";
+
+const cdpPort = process.env.CDP_PORT ?? "9228";
+const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4328";
+const pages = await fetch(`http://127.0.0.1:${cdpPort}/json/list`).then((response) => response.json());
+const page = pages.find((entry) => entry.type === "page");
+if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
+
+const socket = new WebSocket(page.webSocketDebuggerUrl);
+await new Promise((resolve, reject) => {
+  socket.addEventListener("open", resolve, { once: true });
+  socket.addEventListener("error", reject, { once: true });
+});
+
+let nextId = 0;
+const pending = new Map();
+const exceptions = [];
+socket.addEventListener("message", (event) => {
+  const message = JSON.parse(event.data);
+  if (message.id && pending.has(message.id)) {
+    const { resolve, reject } = pending.get(message.id);
+    pending.delete(message.id);
+    if (message.error) reject(new Error(message.error.message));
+    else resolve(message.result);
+  }
+  if (message.method === "Runtime.exceptionThrown") {
+    exceptions.push(message.params.exceptionDetails.exception?.description ?? message.params.exceptionDetails.text);
+  }
+});
+
+const command = (method, params = {}) => new Promise((resolve, reject) => {
+  const id = ++nextId;
+  pending.set(id, { resolve, reject });
+  socket.send(JSON.stringify({ id, method, params }));
+});
+const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
+const evaluate = async (expression) => {
+  const result = await command("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true });
+  if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text);
+  return result.result.value;
+};
+const navigate = async (path) => {
+  await command("Page.navigate", { url: `${baseUrl}${path}` });
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    await pause(100);
+    if (await evaluate("document.readyState === 'complete'")) return;
+  }
+  throw new Error(`${path} 加载超时`);
+};
+const screenshot = async (path) => {
+  const result = await command("Page.captureScreenshot", { format: "png", captureBeyondViewport: false });
+  writeFileSync(path, Buffer.from(result.data, "base64"));
+};
+
+await command("Page.enable");
+await command("Runtime.enable");
+await command("Emulation.setDeviceMetricsOverride", {
+  width: 1440,
+  height: 1100,
+  deviceScaleFactor: 1,
+  mobile: false,
+});
+await navigate("/k3/");
+
+const desktop = await evaluate(`(() => {
+  const root = document.querySelector("[data-gradient-lab]");
+  root.scrollIntoView({ block: "start", behavior: "instant" });
+  window.scrollBy(0, -78);
+  const text = (selector) => root.querySelector(selector)?.textContent.trim();
+  const panel = () => root.querySelector("[data-gradient-panel]:not([hidden])")?.dataset.gradientPanel;
+  const points = (selector) => root.querySelector(selector)?.getAttribute("points");
+  const setSelect = (selector, value) => {
+    const node = root.querySelector(selector);
+    node.value = value;
+    node.dispatchEvent(new Event("change", { bubbles: true }));
+  };
+
+  const initial = {
+    panel: panel(),
+    tabs: root.querySelectorAll("[data-gradient-tab]").length,
+    panels: root.querySelectorAll("[data-gradient-panel]").length,
+    ledger: root.querySelectorAll(".gradient-ledger article").length,
+    definition: root.textContent.includes("论文作者就是这样算的") &&
+      root.textContent.includes("activation gradient") &&
+      root.textContent.includes("parameter gradient"),
+  };
+
+  root.querySelector('[data-gradient-tab="spectrum"]').click();
+  const spectrumInitial = {
+    panel: panel(),
+    baseCv: text("[data-spectrum-base-cv]"),
+    blockCv: text("[data-spectrum-block-cv]"),
+    baseRatio: text("[data-spectrum-base-ratio]"),
+    blockRatio: text("[data-spectrum-block-ratio]"),
+    baseLine: points('[data-chart-line="baseline"]'),
+    blockLine: points('[data-chart-line="block"]'),
+    boundaries: root.querySelectorAll("[data-chart-groups] .group-boundary").length,
+  };
+  root.querySelector('[data-spectrum-scale="normalized"]').click();
+  const normalized = {
+    title: text("[data-spectrum-title]"),
+    baseLine: points('[data-chart-line="baseline"]'),
+  };
+  root.querySelector('[data-spectrum-depth="16"]').click();
+  const depth16 = {
+    baseCv: text("[data-spectrum-base-cv]"),
+    blockCv: text("[data-spectrum-block-cv]"),
+    pointCount: points('[data-chart-line="baseline"]').split(" ").length,
+    boundaries: root.querySelectorAll("[data-chart-groups] .group-boundary").length,
+  };
+  setSelect("[data-spectrum-seed]", "2026073001");
+  setSelect("[data-spectrum-step]", "2000");
+  const seedStep = {
+    state: text("[data-spectrum-state]"),
+    baseCv: text("[data-spectrum-base-cv]"),
+    blockCv: text("[data-spectrum-block-cv]"),
+  };
+
+  root.querySelector('[data-gradient-tab="timeline"]').click();
+  const timelineInitial = {
+    panel: panel(),
+    title: text("[data-time-title]"),
+    line: points('[data-time-line="block"]'),
+    pointCount: root.querySelectorAll('[data-time-points="block"] circle').length,
+  };
+  root.querySelector('[data-time-metric="imbalance"]').click();
+  const timelineChanged = {
+    title: text("[data-time-title]"),
+    line: points('[data-time-line="block"]'),
+  };
+
+  root.querySelector('[data-gradient-tab="output"]').click();
+  const outputInitial = {
+    panel: panel(),
+    pointCount: points('[data-output-line="block"]').split(" ").length,
+    bars: root.querySelectorAll("[data-output-bars] i").length,
+    boundaries: root.querySelectorAll("[data-output-groups] .group-boundary").length,
+    copy: text("[data-output-copy]"),
+  };
+  root.querySelector('[data-output-depth="16"]').click();
+  const outputDepth16 = {
+    pointCount: points('[data-output-line="block"]').split(" ").length,
+    bars: root.querySelectorAll("[data-output-bars] i").length,
+    copy: text("[data-output-copy]"),
+  };
+
+  root.querySelector('[data-gradient-tab="verdict"]').click();
+  const verdict = {
+    panel: panel(),
+    rows: root.querySelectorAll(".verdict-table tbody tr").length,
+    metrics: root.querySelectorAll(".metric-pairs article").length,
+    costs: root.querySelectorAll(".cost-compare article").length,
+    hashes: root.querySelectorAll(".hash-ledger code").length,
+    exact: root.textContent.includes("model + optimizer exact") &&
+      root.textContent.includes("all frozen fields exact"),
+    mixed: root.textContent.includes("depth-dependent or inconclusive"),
+  };
+
+  const first = root.querySelector('[data-gradient-tab="definition"]');
+  first.focus();
+  first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
+  const keyboard = {
+    selected: root.querySelector('[data-gradient-tab][aria-selected="true"]').dataset.gradientTab,
+    panel: panel(),
+  };
+
+  return {
+    initial, spectrumInitial, normalized, depth16, seedStep,
+    timelineInitial, timelineChanged, outputInitial, outputDepth16,
+    verdict, keyboard,
+    documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
+    rootOverflow: root.scrollWidth - root.clientWidth,
+  };
+})()`);
+await pause(180);
+await screenshot("/tmp/llm-atlas-k3-attnres-gradient-desktop.png");
+
+await command("Emulation.setDeviceMetricsOverride", {
+  width: 390,
+  height: 844,
+  deviceScaleFactor: 1,
+  mobile: true,
+});
+await navigate("/k3/");
+const mobile = await evaluate(`(() => {
+  const root = document.querySelector("[data-gradient-lab]");
+  root.scrollIntoView({ block: "start", behavior: "instant" });
+  window.scrollBy(0, -64);
+  root.querySelector('[data-gradient-tab="spectrum"]').click();
+  root.querySelector('[data-spectrum-depth="16"]').click();
+  root.querySelector('[data-gradient-tab="output"]').click();
+  root.querySelector('[data-output-depth="16"]').click();
+  return {
+    tabs: root.querySelectorAll("[data-gradient-tab]").length,
+    ledger: root.querySelectorAll(".gradient-ledger article").length,
+    outputBars: root.querySelectorAll("[data-output-bars] i").length,
+    visiblePanel: root.querySelector("[data-gradient-panel]:not([hidden])")?.dataset.gradientPanel,
+    documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth,
+    rootOverflow: root.scrollWidth - root.clientWidth,
+  };
+})()`);
+await pause(180);
+await screenshot("/tmp/llm-atlas-k3-attnres-gradient-mobile.png");
+
+const report = { desktop, mobile, exceptions };
+console.log(JSON.stringify(report, null, 2));
+
+const numeric = (value) => Number.parseFloat(value.replace("−", "-"));
+const failures = [];
+if (desktop.initial.panel !== "definition" || desktop.initial.tabs !== 5 || desktop.initial.panels !== 5 || desktop.initial.ledger !== 6) failures.push("五视图初始结构异常");
+if (!desktop.initial.definition) failures.push("定义与对象边界缺失");
+if (desktop.spectrumInitial.panel !== "spectrum" || Math.abs(numeric(desktop.spectrumInitial.baseCv) - 0.3786) > 1e-4 || Math.abs(numeric(desktop.spectrumInitial.blockCv) - 0.5996) > 1e-4) failures.push("depth-32 final spectrum 读数异常");
+if (desktop.spectrumInitial.boundaries !== 7 || desktop.spectrumInitial.baseLine === desktop.normalized.baseLine || !desktop.normalized.title.includes("NORMALIZED")) failures.push("绝对/归一化谱或组边界异常");
+if (desktop.depth16.pointCount !== 16 || desktop.depth16.boundaries !== 7 || Math.abs(numeric(desktop.depth16.baseCv) - 0.4246) > 1e-4 || Math.abs(numeric(desktop.depth16.blockCv) - 0.4678) > 1e-4) failures.push("depth-16 spectrum 切换异常");
+if (!desktop.seedStep.state.includes("2026073001") || !desktop.seedStep.state.includes("2,000") || Math.abs(numeric(desktop.seedStep.baseCv) - 0.4101) > 1e-4 || Math.abs(numeric(desktop.seedStep.blockCv) - 0.3656) > 1e-4) failures.push("seed/checkpoint spectrum 切换异常");
+if (desktop.timelineInitial.panel !== "timeline" || desktop.timelineInitial.pointCount !== 6 || desktop.timelineInitial.line === desktop.timelineChanged.line || !desktop.timelineChanged.title.includes("FIRST / LAST")) failures.push("六时点轨迹指标切换异常");
+if (desktop.outputInitial.panel !== "output" || desktop.outputInitial.pointCount !== 32 || desktop.outputInitial.bars !== 32 || desktop.outputInitial.boundaries !== 7 || desktop.outputDepth16.pointCount !== 16 || desktop.outputDepth16.bars !== 16 || !desktop.outputDepth16.copy.includes("DEPTH 16")) failures.push("Output RMS 深度/组节律切换异常");
+if (desktop.verdict.panel !== "verdict" || desktop.verdict.rows !== 2 || desktop.verdict.metrics !== 4 || desktop.verdict.costs !== 3 || desktop.verdict.hashes !== 4 || !desktop.verdict.exact || !desktop.verdict.mixed) failures.push("联合判定、成本或重放视图异常");
+if (desktop.keyboard.selected !== "spectrum" || desktop.keyboard.panel !== "spectrum") failures.push("键盘 tab 导航异常");
+if (desktop.documentOverflow > 1 || desktop.rootOverflow > 1 || mobile.documentOverflow > 1 || mobile.rootOverflow > 1) failures.push("桌面或移动端出现文档级横向溢出");
+if (mobile.tabs !== 5 || mobile.ledger !== 6 || mobile.outputBars !== 16 || mobile.visiblePanel !== "output") failures.push("移动端交互结构异常");
+if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);
+
+if (failures.length) {
+  console.error(`\nFAIL\n- ${failures.join("\n- ")}`);
+  process.exitCode = 1;
+} else {
+  console.log("\nPASS K3 AttnRes gradient browser regression");
+}
+
+socket.close();
@@ -0,0 +1,68 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync } from "node:fs";
+
+const read = (path) => {
+  const bytes = readFileSync(new URL(path, import.meta.url));
+  return {
+    bytes,
+    json: JSON.parse(bytes),
+    sha256: createHash("sha256").update(bytes).digest("hex"),
+  };
+};
+
+const raw = read("../src/data/k3-attnres-gradient.json");
+const compact = read("../src/data/k3-attnres-gradient-compact.json");
+const reproduction = read("../experiments/k3/attnres_gradient/reproduction.json");
+const rawDirectory = new URL("../experiments/k3/attnres_gradient/results/raw/", import.meta.url);
+const failures = [];
+const expect = (condition, message) => {
+  if (!condition) failures.push(message);
+};
+
+expect(raw.sha256 === "ad461cbe74fc671f356288d37c9b628618814915116e4fbe03aaaa48367e6a8d", "raw aggregate SHA-256 changed");
+expect(compact.sha256 === "5377a5e731db2fdc85a0327f05d43f1cd067d3fc34633619d3004f3fc31968e3", "compact payload SHA-256 changed");
+expect(reproduction.sha256 === "aedcde6accc6eb1c24b122ffb55706ef620062e848d9fe4c5f622b084a4fdea6", "reproduction payload SHA-256 changed");
+expect(compact.json.protocol_id === "llm-atlas-k3-attnres-gradient-scale-v1", "protocol identity mismatch");
+expect(compact.json.study.formal_runs === 12, "formal grid is not 12 cells");
+expect(compact.json.study.formal_target_bytes === 786_432_000, "formal target-byte budget mismatch");
+expect(compact.json.study.replay_target_bytes === 65_536_000, "replay byte budget mismatch");
+expect(compact.json.cells.length === 12, "compact cell count mismatch");
+expect(Object.keys(raw.json.runs).length === 12, "raw formal cell count mismatch");
+expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 21, "public raw output count is not 21");
+expect(reproduction.json.replay_exact.exact, "full formal replay is not exact");
+expect(Object.values(reproduction.json.smoke_exact).every((row) => row.exact && row.gradient_gate.passed), "paired smoke or loss-scale gate failed");
+
+for (const depth of ["16", "32"]) {
+  const summary = compact.json.depth_summaries[depth];
+  expect(summary.verdict.label === "mixed / inconclusive at this depth", `depth ${depth} verdict changed`);
+  expect(summary.by_seed.length === 3, `depth ${depth} seed count changed`);
+  expect(summary.by_seed.every((row) => row.block_minus_baseline_bpc < 0), `depth ${depth} BPC pairing changed`);
+  expect(summary.by_seed.every((row) => row.relative_cv_reduction < 0), `depth ${depth} CV counterevidence changed`);
+  expect(summary.by_seed.every((row) => row.relative_imbalance_reduction > 0), `depth ${depth} first/last improvement changed`);
+}
+
+expect(Math.abs(compact.json.depth_summaries["16"].means.relative_cv_reduction - (-0.10264135379685868)) < 1e-15, "depth-16 CV contrast changed");
+expect(Math.abs(compact.json.depth_summaries["32"].means.relative_cv_reduction - (-0.600330100169428)) < 1e-15, "depth-32 CV contrast changed");
+expect(Math.abs(compact.json.depth_summaries["16"].means.relative_imbalance_reduction - 0.6099652484299795) < 1e-15, "depth-16 imbalance contrast changed");
+expect(Math.abs(compact.json.depth_summaries["32"].means.relative_imbalance_reduction - 0.7200517407719272) < 1e-15, "depth-32 imbalance contrast changed");
+expect(compact.json.overall_verdict === "depth-dependent or inconclusive", "overall preregistered verdict changed");
+
+if (failures.length) {
+  console.error(`FAIL K3 AttnRes gradient data\n- ${failures.join("\n- ")}`);
+  process.exit(1);
+}
+
+console.log(JSON.stringify({
+  protocol: compact.json.protocol_id,
+  formalRuns: compact.json.study.formal_runs,
+  formalTargetBytes: compact.json.study.formal_target_bytes,
+  depth16: compact.json.depth_summaries["16"].verdict.label,
+  depth32: compact.json.depth_summaries["32"].verdict.label,
+  replayExact: reproduction.json.replay_exact.exact,
+  hashes: {
+    raw: raw.sha256,
+    compact: compact.sha256,
+    reproduction: reproduction.sha256,
+  },
+}, null, 2));
+console.log("PASS K3 AttnRes gradient frozen data");
@@ -81,6 +81,8 @@ const overview = await evaluate(`(() => ({
  artifactMismatch: document.querySelector("#artifacts")?.textContent.includes("A_log [128] ≠ expected [96]"),
  attnresTabs: document.querySelectorAll("[data-attnres-tab]").length,
  attnresPanels: document.querySelectorAll("[data-attnres-panel]").length,
+  gradientTabs: document.querySelectorAll("[data-gradient-tab]").length,
+  gradientPanels: document.querySelectorAll("[data-gradient-panel]").length,
  nativeVisionCorrected: document.body.textContent.includes("MoonViT‑V2 从头训练") &&
    document.body.textContent.includes("同一个 next-token prediction objective"),
  staleVisionClaim: document.body.textContent.includes("先固定语言模型训练视觉组件"),
@@ -290,8 +292,9 @@ const mobile = await evaluate(`(() => {
    artifactTabs: document.querySelectorAll("[data-artifact-tab]").length,
    artifactLayers: document.querySelectorAll("[data-layer-cell]").length,
    attnresTabs: document.querySelectorAll("[data-attnres-tab]").length,
+    gradientTabs: document.querySelectorAll("[data-gradient-tab]").length,
    offenders: [...document.querySelectorAll("body *")]
-      .filter((node) => !node.closest(".paper-chain, .spec-table-wrap, .cache-strip, .architecture-explorer, [data-k3-lab], [data-k3-artifact-lab], [data-attnres-lab]"))
+      .filter((node) => !node.closest(".paper-chain, .spec-table-wrap, .cache-strip, .architecture-explorer, [data-k3-lab], [data-k3-artifact-lab], [data-attnres-lab], [data-gradient-lab]"))
      .filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
      .slice(0, 15)
      .map((node) => ({
@@ -322,12 +325,13 @@ console.log(JSON.stringify(report, null, 2));
 const numeric = (text) => Number.parseFloat(text.replaceAll(",", "").replace("−", "-"));
 const failures = [];
 if (!overview.title.includes("因果环节")) failures.push("K3 二轮标题异常");
-if (overview.sections !== 33 || overview.tocLinks !== 33) failures.push("32 个编号专题加阅读链的目录结构异常");
+if (overview.sections !== 34 || overview.tocLinks !== 34) failures.push("33 个编号专题加阅读链的目录结构异常");
 if (overview.ledgers !== 32 || overview.reportMap !== 9) failures.push("32 张问题账或报告地图异常");
 if (overview.figureAtlas !== 21 || overview.paperLinks !== 100 || overview.paperGroups < 12) failures.push("图表审计或 100 节点阅读链异常");
 if (overview.labTabs !== 8 || overview.labPanels !== 8) failures.push("八联实验结构异常");
 if (overview.artifactTabs !== 4 || overview.artifactPanels !== 4 || overview.artifactLayers !== 93 || !overview.artifactMismatch) failures.push("开放工件四视图、93 层条带或形状冲突异常");
 if (overview.attnresTabs !== 5 || overview.attnresPanels !== 5) failures.push("AttnRes 独立实验五视图异常");
+if (overview.gradientTabs !== 5 || overview.gradientPanels !== 5) failures.push("AttnRes 梯度定义扩展五视图异常");
 if (!overview.nativeVisionCorrected || overview.staleVisionClaim) failures.push("原生多模态纠错未生效或旧错误残留");
 if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在文档级横向溢出");
 if (labs.memoryInitial.panel !== "memory" || numeric(labs.memoryInitial.additiveError) <= numeric(labs.memoryInitial.deltaError)) failures.push("Delta memory 初始递推异常");
@@ -352,7 +356,7 @@ if (artifacts.parameterChanged.shape !== "[96,128] F32" || !artifacts.parameterC
 if (artifacts.reproductionInitial.panel !== "reproduction" || numeric(artifacts.reproductionInitial.speedup) !== 1.85 || numeric(artifacts.reproductionInitial.localMean) < 2.6 || !artifacts.reproductionInitial.exactSuite || numeric(artifacts.reproductionInitial.cv) < 2) failures.push("FlashKDA H20、本机 exact suite 或 router 初始探针异常");
 if (numeric(artifacts.reproductionChanged.speedup) !== 3.27 || numeric(artifacts.reproductionChanged.flash) !== 0.7064 || numeric(artifacts.reproductionChanged.localMean) >= numeric(artifacts.reproductionInitial.localMean) || !artifacts.reproductionChanged.localMode.includes("FP32 state") || numeric(artifacts.reproductionChanged.cv) <= numeric(artifacts.reproductionInitial.cv) || numeric(artifacts.reproductionChanged.zero) <= numeric(artifacts.reproductionInitial.zero)) failures.push("GB200 benchmark、本机 varlen/state 或 synthetic router counterexample 未更新");
 if (artifacts.keyboardSelected !== "tensors" || artifacts.keyboardVisible !== "tensors") failures.push("开放工件键盘 tab 导航异常");
-if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 8 || mobile.artifactTabs !== 4 || mobile.artifactLayers !== 93 || mobile.attnresTabs !== 5) failures.push("移动端导航或实验异常");
+if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 8 || mobile.artifactTabs !== 4 || mobile.artifactLayers !== 93 || mobile.attnresTabs !== 5 || mobile.gradientTabs !== 5) failures.push("移动端导航或实验异常");
 if (mobile.offenders.length) failures.push(`移动端越界元素：${JSON.stringify(mobile.offenders)}`);
 if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);