feat: map DeepSeek Chat sampling robustness

2026-07-30 02:01:50 +08:00
parent 18b16e2fdc
commit 580f69675c
21 changed files with 108346 additions and 28 deletions
@@ -0,0 +1,278 @@
+import { writeFileSync } from "node:fs";
+
+const cdpPort = process.env.CDP_PORT ?? "9230";
+const baseUrl = process.env.SITE_URL ?? "http://127.0.0.1:4327";
+const pages = await fetch(
+  `http://127.0.0.1:${cdpPort}/json/list`,
+).then((response) => response.json());
+const page = pages.find((entry) => entry.type === "page");
+if (!page) throw new Error(`CDP ${cdpPort} 没有可用页面`);
+
+const socket = new WebSocket(page.webSocketDebuggerUrl);
+await new Promise((resolve, reject) => {
+  socket.addEventListener("open", resolve, { once: true });
+  socket.addEventListener("error", reject, { once: true });
+});
+
+let nextId = 0;
+const pending = new Map();
+const exceptions = [];
+socket.addEventListener("message", (event) => {
+  const message = JSON.parse(event.data);
+  if (message.id && pending.has(message.id)) {
+    const { resolve, reject } = pending.get(message.id);
+    pending.delete(message.id);
+    if (message.error) reject(new Error(message.error.message));
+    else resolve(message.result);
+  }
+  if (message.method === "Runtime.exceptionThrown") {
+    exceptions.push(
+      message.params.exceptionDetails.exception?.description
+      ?? message.params.exceptionDetails.text,
+    );
+  }
+});
+
+const command = (method, params = {}) => new Promise((resolve, reject) => {
+  const id = ++nextId;
+  pending.set(id, { resolve, reject });
+  socket.send(JSON.stringify({ id, method, params }));
+});
+const pause = (milliseconds) => new Promise(
+  (resolve) => setTimeout(resolve, milliseconds),
+);
+const evaluate = async (expression) => {
+  const result = await command("Runtime.evaluate", {
+    expression,
+    returnByValue: true,
+    awaitPromise: true,
+  });
+  if (result.exceptionDetails) {
+    throw new Error(
+      result.exceptionDetails.exception?.description
+      ?? result.exceptionDetails.text,
+    );
+  }
+  return result.result.value;
+};
+const navigate = async (path) => {
+  await command("Page.navigate", { url: `${baseUrl}${path}` });
+  for (let attempt = 0; attempt < 80; attempt += 1) {
+    await pause(100);
+    if (await evaluate("document.readyState === 'complete'")) return;
+  }
+  throw new Error(`${path} 加载超时`);
+};
+const screenshot = async (path) => {
+  const result = await command("Page.captureScreenshot", {
+    format: "png",
+    captureBeyondViewport: false,
+  });
+  writeFileSync(path, Buffer.from(result.data, "base64"));
+};
+const assert = (condition, message) => {
+  if (!condition) throw new Error(message);
+};
+
+await command("Page.enable");
+await command("Runtime.enable");
+await command("Emulation.setDeviceMetricsOverride", {
+  width: 1440,
+  height: 1100,
+  deviceScaleFactor: 1,
+  mobile: false,
+});
+await navigate("/deepseek/");
+
+const overview = await evaluate(`(() => {
+  const root = document.querySelector("[data-sampling-lab]");
+  document.documentElement.style.scrollBehavior = "auto";
+  window.scrollTo(0, root.getBoundingClientRect().top + window.scrollY);
+  return {
+    tabs: root.querySelectorAll("[data-sp-tab]").length,
+    panels: root.querySelectorAll("[data-sp-panel]").length,
+    seedBars: root.querySelectorAll("[data-sp-seeds] button").length,
+    conditions: root.querySelectorAll(".condition-overview > div").length,
+    edges: root.querySelectorAll(".edge-overview > div").length,
+    heroLabs: [...document.querySelectorAll(".page-facts > div")]
+      .find((node) => node.querySelector("dt")?.textContent.trim() === "LABS")
+      ?.querySelector("dd")?.textContent.trim(),
+    activePanel: root.querySelector("[data-sp-panel]:not([hidden])")
+      ?.dataset.spPanel,
+    unique: root.querySelector("[data-sp-unique]").textContent.trim(),
+    eos: root.querySelector("[data-sp-eos]").textContent.trim(),
+    greedy: root.querySelector("[data-sp-greedy]").textContent.trim(),
+    documentOverflow: (
+      document.documentElement.scrollWidth
+      - document.documentElement.clientWidth
+    ),
+  };
+})()`);
+await pause(300);
+await screenshot("/tmp/llm-atlas-sampling-desktop.png");
+
+const trajectorySwitch = await evaluate(`(() => {
+  const root = document.querySelector("[data-sampling-lab]");
+  const source = root.querySelector("[data-sp-source]");
+  const condition = root.querySelector("[data-sp-condition]");
+  source.value = "HumanEval/31";
+  source.dispatchEvent(new Event("change", { bubbles: true }));
+  condition.value = "s1_period";
+  condition.dispatchEvent(new Event("change", { bubbles: true }));
+  const buttons = [...root.querySelectorAll("[data-sp-seeds] button")];
+  buttons.at(-1).click();
+  return {
+    unique: root.querySelector("[data-sp-unique]").textContent.trim(),
+    eos: root.querySelector("[data-sp-eos]").textContent.trim(),
+    greedy: root.querySelector("[data-sp-greedy]").textContent.trim(),
+    seedBars: buttons.length,
+    repeats: buttons.filter((button) => button.classList.contains("repeat")).length,
+    selected: buttons.filter((button) => button.classList.contains("selected")).length,
+    focus: root.querySelector("[data-sp-focus-seed]").textContent.trim(),
+    hash: root.querySelector("[data-sp-focus-hash]").textContent.trim(),
+  };
+})()`);
+
+const taskSwitch = await evaluate(`(() => {
+  const root = document.querySelector("[data-sampling-lab]");
+  root.querySelector('[data-sp-tab="tasks"]').click();
+  const condition = root.querySelector("[data-sp-task-condition]");
+  const read = () => ({
+    panel: root.querySelector("[data-sp-panel]:not([hidden])").dataset.spPanel,
+    math: root.querySelector("[data-sp-math-total]").textContent.trim(),
+    code: root.querySelector("[data-sp-code-total]").textContent.trim(),
+    mathCells: root.querySelectorAll("[data-sp-math-cells] article").length,
+    codeCells: root.querySelectorAll("[data-sp-code-cells] article").length,
+  });
+  condition.value = "s1_x";
+  condition.dispatchEvent(new Event("change", { bubbles: true }));
+  const systemX = read();
+  condition.value = "s0_period";
+  condition.dispatchEvent(new Event("change", { bubbles: true }));
+  const period = read();
+  return { systemX, period };
+})()`);
+
+const edgeSwitch = await evaluate(`(() => {
+  const root = document.querySelector("[data-sampling-lab]");
+  root.querySelector('[data-sp-tab="edges"]').click();
+  const source = root.querySelector("[data-sp-edge-source]");
+  const edge = root.querySelector("[data-sp-edge]");
+  source.value = "gsm8k/test/1069";
+  source.dispatchEvent(new Event("change", { bubbles: true }));
+  edge.value = "system_eos";
+  edge.dispatchEvent(new Event("change", { bubbles: true }));
+  return {
+    panel: root.querySelector("[data-sp-panel]:not([hidden])").dataset.spPanel,
+    aligned: root.querySelector("[data-sp-edge-aligned]").textContent.trim(),
+    nearest: root.querySelector("[data-sp-edge-nearest]").textContent.trim(),
+    overlap: root.querySelector("[data-sp-edge-overlap]").textContent.trim(),
+    left: root.querySelectorAll("[data-sp-edge-left-set] article").length,
+    right: root.querySelectorAll("[data-sp-edge-right-set] article").length,
+    highlighted: root.querySelectorAll(".two-sets article.overlap").length,
+  };
+})()`);
+
+const keyboard = await evaluate(`(() => {
+  const root = document.querySelector("[data-sampling-lab]");
+  const first = root.querySelector('[data-sp-tab="trajectories"]');
+  first.click();
+  first.focus();
+  first.dispatchEvent(new KeyboardEvent("keydown", {
+    key: "ArrowRight",
+    bubbles: true,
+  }));
+  const selected = root.querySelector(
+    '[data-sp-tab][aria-selected="true"]',
+  ).dataset.spTab;
+  root.querySelector('[data-sp-tab="reproduction"]').click();
+  return {
+    keyboardSelected: selected,
+    visible: root.querySelector("[data-sp-panel]:not([hidden])")
+      .dataset.spPanel,
+    reproFields: root.querySelectorAll(".repro-fields article").length,
+  };
+})()`);
+
+await command("Emulation.setDeviceMetricsOverride", {
+  width: 390,
+  height: 844,
+  deviceScaleFactor: 1,
+  mobile: true,
+});
+await navigate("/deepseek/#sampling");
+const mobile = await evaluate(`(() => {
+  const root = document.querySelector("[data-sampling-lab]");
+  document.documentElement.style.scrollBehavior = "auto";
+  window.scrollTo(0, root.getBoundingClientRect().top + window.scrollY);
+  return {
+    tabs: root.querySelectorAll("[data-sp-tab]").length,
+    seedBars: root.querySelectorAll("[data-sp-seeds] button").length,
+    documentOverflow: (
+      document.documentElement.scrollWidth
+      - document.documentElement.clientWidth
+    ),
+    rootOverflow: root.scrollWidth - root.clientWidth,
+  };
+})()`);
+await pause(300);
+await screenshot("/tmp/llm-atlas-sampling-mobile.png");
+
+assert(overview.tabs === 4, `sampling tabs=${overview.tabs}`);
+assert(overview.panels === 4, `sampling panels=${overview.panels}`);
+assert(overview.seedBars === 8, `initial seed bars=${overview.seedBars}`);
+assert(overview.conditions === 8, `condition rows=${overview.conditions}`);
+assert(overview.edges === 10, `edge rows=${overview.edges}`);
+assert(overview.heroLabs?.startsWith("20"), `hero labs=${overview.heroLabs}`);
+assert(overview.activePanel === "trajectories", `active=${overview.activePanel}`);
+assert(overview.unique === "8 / 8", `initial unique=${overview.unique}`);
+assert(overview.eos === "5 / 8", `initial eos=${overview.eos}`);
+assert(overview.greedy === "NO", `initial greedy=${overview.greedy}`);
+assert(overview.documentOverflow === 0, `desktop overflow=${overview.documentOverflow}`);
+
+assert(trajectorySwitch.unique === "2 / 8", `switched unique=${trajectorySwitch.unique}`);
+assert(trajectorySwitch.eos === "8 / 8", `switched eos=${trajectorySwitch.eos}`);
+assert(trajectorySwitch.greedy === "YES", `switched greedy=${trajectorySwitch.greedy}`);
+assert(trajectorySwitch.seedBars === 8, `switched seeds=${trajectorySwitch.seedBars}`);
+assert(trajectorySwitch.repeats >= 2, `repeat markers=${trajectorySwitch.repeats}`);
+assert(trajectorySwitch.selected === 1, `selected seeds=${trajectorySwitch.selected}`);
+assert(trajectorySwitch.focus.startsWith("R7"), `focus=${trajectorySwitch.focus}`);
+assert(trajectorySwitch.hash.includes("sha256"), `focus hash=${trajectorySwitch.hash}`);
+
+assert(taskSwitch.systemX.panel === "tasks", `task panel=${taskSwitch.systemX.panel}`);
+assert(taskSwitch.systemX.math === "7 / 8 PASS", `system x math=${taskSwitch.systemX.math}`);
+assert(taskSwitch.systemX.code === "8 / 8 PASS", `system x code=${taskSwitch.systemX.code}`);
+assert(taskSwitch.period.code === "7 / 8 PASS", `period code=${taskSwitch.period.code}`);
+assert(taskSwitch.period.mathCells === 8, `math cells=${taskSwitch.period.mathCells}`);
+assert(taskSwitch.period.codeCells === 8, `code cells=${taskSwitch.period.codeCells}`);
+
+assert(edgeSwitch.panel === "edges", `edge panel=${edgeSwitch.panel}`);
+assert(edgeSwitch.aligned === "77.3%", `edge aligned=${edgeSwitch.aligned}`);
+assert(edgeSwitch.nearest === "89.4%", `edge nearest=${edgeSwitch.nearest}`);
+assert(edgeSwitch.overlap === "1 / 14", `edge overlap=${edgeSwitch.overlap}`);
+assert(edgeSwitch.left === 8 && edgeSwitch.right === 8, `edge sets=${edgeSwitch.left}/${edgeSwitch.right}`);
+assert(edgeSwitch.highlighted >= 2, `edge highlights=${edgeSwitch.highlighted}`);
+
+assert(keyboard.keyboardSelected === "tasks", `keyboard selected=${keyboard.keyboardSelected}`);
+assert(keyboard.visible === "reproduction", `repro visible=${keyboard.visible}`);
+assert(keyboard.reproFields === 8, `repro fields=${keyboard.reproFields}`);
+assert(mobile.tabs === 4 && mobile.seedBars === 8, `mobile controls=${mobile.tabs}/${mobile.seedBars}`);
+assert(mobile.documentOverflow === 0, `mobile document overflow=${mobile.documentOverflow}`);
+assert(mobile.rootOverflow === 0, `mobile root overflow=${mobile.rootOverflow}`);
+assert(exceptions.length === 0, `runtime exceptions: ${exceptions.join(" | ")}`);
+
+console.log(JSON.stringify({
+  overview,
+  trajectorySwitch,
+  taskSwitch,
+  edgeSwitch,
+  keyboard,
+  mobile,
+  exceptions,
+  screenshots: [
+    "/tmp/llm-atlas-sampling-desktop.png",
+    "/tmp/llm-atlas-sampling-mobile.png",
+  ],
+}, null, 2));
+
+socket.close();