feat: trace DeepSeek Chat completion depth
This commit is contained in:
wuyang
@@ -0,0 +1,461 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
const root = resolve(import.meta.dirname, "..");
|
||||
const paths = {
|
||||
baseline: resolve(root, "src/data/deepseek-v2-lite-chat-behavior.json"),
|
||||
completion: resolve(
|
||||
root,
|
||||
"src/data/deepseek-v2-lite-chat-completion-512.json",
|
||||
),
|
||||
completionEval: resolve(
|
||||
root,
|
||||
"src/data/deepseek-v2-lite-chat-completion-512-eval.json",
|
||||
),
|
||||
completionRepro: resolve(
|
||||
root,
|
||||
"src/data/deepseek-v2-lite-chat-completion-512-repro-1pd.json",
|
||||
),
|
||||
depth: resolve(
|
||||
root,
|
||||
"src/data/deepseek-v2-lite-chat-full-depth.json",
|
||||
),
|
||||
depthRepro: resolve(
|
||||
root,
|
||||
"src/data/deepseek-v2-lite-chat-full-depth-repro-1pd.json",
|
||||
),
|
||||
output: resolve(
|
||||
root,
|
||||
"src/data/deepseek-v2-lite-chat-completion-depth-compact.json",
|
||||
),
|
||||
};
|
||||
|
||||
const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
|
||||
const sha256 = (path) => createHash("sha256")
|
||||
.update(readFileSync(path))
|
||||
.digest("hex");
|
||||
const artifact = (path) => ({
|
||||
bytes: statSync(path).size,
|
||||
sha256: sha256(path),
|
||||
});
|
||||
|
||||
const baseline = readJson(paths.baseline);
|
||||
const completion = readJson(paths.completion);
|
||||
const completionEval = readJson(paths.completionEval);
|
||||
const completionRepro = readJson(paths.completionRepro);
|
||||
const depth = readJson(paths.depth);
|
||||
const depthRepro = readJson(paths.depthRepro);
|
||||
|
||||
const completionArtifact = artifact(paths.completion);
|
||||
const baselineArtifact = artifact(paths.baseline);
|
||||
if (
|
||||
completionEval.input.behavior_sha256 !== completionArtifact.sha256
|
||||
|| completionEval.input.baseline_sha256 !== baselineArtifact.sha256
|
||||
) {
|
||||
throw new Error("completion evaluator input hash contract failed");
|
||||
}
|
||||
|
||||
const outputKey = (sourceId, condition) => `${sourceId}\0${condition}`;
|
||||
const completionByKey = new Map(
|
||||
completion.sources.flatMap((source) => source.outputs.map((output) => [
|
||||
outputKey(source.id, output.condition),
|
||||
output,
|
||||
])),
|
||||
);
|
||||
const completionReproduction = {
|
||||
sources: completionRepro.sources.length,
|
||||
cells: 0,
|
||||
promptHashExact: 0,
|
||||
generatedTokenIdsExact: 0,
|
||||
generatedTextExact: 0,
|
||||
eosStateExact: 0,
|
||||
truncationStateExact: 0,
|
||||
};
|
||||
for (const source of completionRepro.sources) {
|
||||
for (const output of source.outputs) {
|
||||
const formal = completionByKey.get(outputKey(source.id, output.condition));
|
||||
if (!formal) {
|
||||
throw new Error(
|
||||
`completion formal output missing: ${source.id}/${output.condition}`,
|
||||
);
|
||||
}
|
||||
completionReproduction.cells += 1;
|
||||
completionReproduction.promptHashExact += (
|
||||
formal.prompt_token_ids_sha256 === output.prompt_token_ids_sha256
|
||||
);
|
||||
completionReproduction.generatedTokenIdsExact += (
|
||||
JSON.stringify(formal.generated_token_ids)
|
||||
=== JSON.stringify(output.generated_token_ids)
|
||||
);
|
||||
completionReproduction.generatedTextExact += formal.text === output.text;
|
||||
completionReproduction.eosStateExact += formal.hit_eos === output.hit_eos;
|
||||
completionReproduction.truncationStateExact += (
|
||||
formal.stopped_at_max_new_tokens
|
||||
=== output.stopped_at_max_new_tokens
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const [name, value] of Object.entries(completionReproduction)) {
|
||||
if (
|
||||
!["sources", "cells"].includes(name)
|
||||
&& value !== completionReproduction.cells
|
||||
) {
|
||||
throw new Error(
|
||||
`completion reproduction mismatch: ${name}=${value}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const stableDepthSource = (source) => {
|
||||
const copy = structuredClone(source);
|
||||
delete copy.forward_seconds;
|
||||
delete copy.peak_cuda_memory_allocated_bytes;
|
||||
return copy;
|
||||
};
|
||||
const depthSourceById = new Map(
|
||||
depth.sources.map((source) => [source.id, source]),
|
||||
);
|
||||
const depthReproduction = {
|
||||
sources: depthRepro.sources.length,
|
||||
sourceObjectsExactAfterRuntimeStrip: 0,
|
||||
hiddenStagesPerSource: 29,
|
||||
routerLayersPerSource: 26,
|
||||
hiddenTensorHashesExact: 0,
|
||||
hiddenTensorHashesCompared: 0,
|
||||
orderedRouteHashesExact: 0,
|
||||
orderedRouteHashesCompared: 0,
|
||||
routeWeightHashesExact: 0,
|
||||
routeWeightHashesCompared: 0,
|
||||
};
|
||||
for (const rerunSource of depthRepro.sources) {
|
||||
const formalSource = depthSourceById.get(rerunSource.id);
|
||||
if (!formalSource) {
|
||||
throw new Error(`depth formal source missing: ${rerunSource.id}`);
|
||||
}
|
||||
depthReproduction.sourceObjectsExactAfterRuntimeStrip += (
|
||||
JSON.stringify(stableDepthSource(formalSource))
|
||||
=== JSON.stringify(stableDepthSource(rerunSource))
|
||||
);
|
||||
for (const [stage, stageValue] of Object.entries(
|
||||
rerunSource.hidden_stages,
|
||||
)) {
|
||||
for (const [condition, conditionValue] of Object.entries(
|
||||
stageValue.conditions,
|
||||
)) {
|
||||
for (const scope of ["target_content", "full_input"]) {
|
||||
depthReproduction.hiddenTensorHashesCompared += 1;
|
||||
depthReproduction.hiddenTensorHashesExact += (
|
||||
formalSource.hidden_stages[stage].conditions[condition][scope]
|
||||
.tensor_sha256
|
||||
=== conditionValue[scope].tensor_sha256
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [layer, layerValue] of Object.entries(
|
||||
rerunSource.router_layers,
|
||||
)) {
|
||||
for (const [condition, conditionValue] of Object.entries(
|
||||
layerValue.conditions,
|
||||
)) {
|
||||
for (const scope of ["target_content", "full_input"]) {
|
||||
const formalScope = (
|
||||
formalSource.router_layers[layer].conditions[condition][scope]
|
||||
);
|
||||
depthReproduction.orderedRouteHashesCompared += 1;
|
||||
depthReproduction.orderedRouteHashesExact += (
|
||||
formalScope.ordered_route_sha256
|
||||
=== conditionValue[scope].ordered_route_sha256
|
||||
);
|
||||
depthReproduction.routeWeightHashesCompared += 1;
|
||||
depthReproduction.routeWeightHashesExact += (
|
||||
formalScope.route_weight_sha256
|
||||
=== conditionValue[scope].route_weight_sha256
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
depthReproduction.sourceObjectsExactAfterRuntimeStrip
|
||||
!== depthReproduction.sources
|
||||
|| depthReproduction.hiddenTensorHashesExact
|
||||
!== depthReproduction.hiddenTensorHashesCompared
|
||||
|| depthReproduction.orderedRouteHashesExact
|
||||
!== depthReproduction.orderedRouteHashesCompared
|
||||
|| depthReproduction.routeWeightHashesExact
|
||||
!== depthReproduction.routeWeightHashesCompared
|
||||
) {
|
||||
throw new Error(
|
||||
`depth reproduction mismatch: ${JSON.stringify(depthReproduction)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const conditions = Object.keys(completion.generation_contract.conditions);
|
||||
const edgeOrder = Object.keys(depth.trace_contract.edge_pairs);
|
||||
const edgeLabels = {
|
||||
system_eos: "System on − off · EOS",
|
||||
system_bos: "System on − off · BOS",
|
||||
system_x: "System on − off · x",
|
||||
system_period: "System on − off · 句点",
|
||||
bos_at_s0: "BOS − EOS · system off",
|
||||
bos_at_s1: "BOS − EOS · system on",
|
||||
x_at_s0: "x − EOS · system off",
|
||||
x_at_s1: "x − EOS · system on",
|
||||
period_at_s0: "句点 − EOS · system off",
|
||||
period_at_s1: "句点 − EOS · system on",
|
||||
};
|
||||
const domains = ["all", "english", "chinese", "code", "math"];
|
||||
const hiddenStages = [
|
||||
"embedding",
|
||||
...Array.from({ length: 27 }, (_, index) => (
|
||||
`layer_${String(index).padStart(2, "0")}`
|
||||
)),
|
||||
"final_norm",
|
||||
];
|
||||
const routerLayers = Array.from({ length: 26 }, (_, index) => (
|
||||
`layer_${String(index + 1).padStart(2, "0")}`
|
||||
));
|
||||
|
||||
const sourcesForDomain = (domain) => (
|
||||
domain === "all"
|
||||
? depth.sources
|
||||
: depth.sources.filter((source) => source.domain === domain)
|
||||
);
|
||||
const mean = (values) => (
|
||||
values.reduce((sum, value) => sum + value, 0) / values.length
|
||||
);
|
||||
const aggregateHidden = (sources, stage, edge) => {
|
||||
const rows = sources.map(
|
||||
(source) => (
|
||||
source.hidden_stages[stage].target_comparisons[edge]
|
||||
),
|
||||
);
|
||||
const tokens = rows.reduce((sum, row) => sum + row.tokens, 0);
|
||||
return {
|
||||
tokens,
|
||||
exactRows: rows.reduce(
|
||||
(sum, row) => sum + row.exact_hidden_rows,
|
||||
0,
|
||||
),
|
||||
meanCosine: rows.reduce(
|
||||
(sum, row) => sum + row.mean_cosine_similarity * row.tokens,
|
||||
0,
|
||||
) / tokens,
|
||||
meanRelativeL2: mean(rows.map((row) => row.mean_relative_l2)),
|
||||
maxAbsDelta: Math.max(...rows.map((row) => row.max_abs_delta)),
|
||||
};
|
||||
};
|
||||
const aggregateRouter = (sources, layer, edge) => {
|
||||
const rows = sources.map(
|
||||
(source) => (
|
||||
source.router_layers[layer].target_comparisons[edge]
|
||||
),
|
||||
);
|
||||
const tokens = rows.reduce((sum, row) => sum + row.tokens, 0);
|
||||
const orderedExact = rows.reduce(
|
||||
(sum, row) => sum + row.ordered_topk_exact_tokens,
|
||||
0,
|
||||
);
|
||||
const setExact = rows.reduce(
|
||||
(sum, row) => sum + row.set_exact_tokens,
|
||||
0,
|
||||
);
|
||||
return {
|
||||
tokens,
|
||||
orderedExact,
|
||||
orderedExactRate: orderedExact / tokens,
|
||||
setExact,
|
||||
setExactRate: setExact / tokens,
|
||||
meanSetJaccard: rows.reduce(
|
||||
(sum, row) => sum + row.mean_set_jaccard * row.tokens,
|
||||
0,
|
||||
) / tokens,
|
||||
meanTokenWeightedTv: mean(
|
||||
rows.map((row) => row.mean_token_weighted_tv),
|
||||
),
|
||||
meanAggregateLoadTv: mean(
|
||||
rows.map((row) => row.aggregate_load_tv),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
const hiddenSeries = Object.fromEntries(domains.map((domain) => {
|
||||
const sources = sourcesForDomain(domain);
|
||||
return [domain, Object.fromEntries(edgeOrder.map((edge) => [
|
||||
edge,
|
||||
hiddenStages.map((stage) => ({
|
||||
stage,
|
||||
...aggregateHidden(sources, stage, edge),
|
||||
})),
|
||||
]))];
|
||||
}));
|
||||
const routerSeries = Object.fromEntries(domains.map((domain) => {
|
||||
const sources = sourcesForDomain(domain);
|
||||
return [domain, Object.fromEntries(edgeOrder.map((edge) => [
|
||||
edge,
|
||||
routerLayers.map((layer) => ({
|
||||
layer,
|
||||
...aggregateRouter(sources, layer, edge),
|
||||
})),
|
||||
]))];
|
||||
}));
|
||||
|
||||
const evaluationRows = completionEval.rows.map((row) => ({
|
||||
sourceId: row.source_id,
|
||||
domain: row.domain,
|
||||
condition: row.condition,
|
||||
generatedTokens: row.generated_tokens,
|
||||
hitEos: row.hit_eos,
|
||||
completionClass: row.completion_class,
|
||||
taskEvaluation: row.task_evaluation,
|
||||
}));
|
||||
const incompleteRows = evaluationRows.filter((row) => !row.hitEos);
|
||||
const codeRows = evaluationRows.filter((row) => row.domain === "code");
|
||||
const mathRows = evaluationRows.filter((row) => row.domain === "math");
|
||||
const codeStatuses = Object.groupBy(
|
||||
codeRows,
|
||||
(row) => row.taskEvaluation.execution?.status ?? "not_run",
|
||||
);
|
||||
const mathExtraction = Object.groupBy(
|
||||
mathRows,
|
||||
(row) => row.taskEvaluation.extraction_method,
|
||||
);
|
||||
|
||||
const baselineNaturalEos = baseline.sources.reduce(
|
||||
(sum, source) => (
|
||||
sum + source.outputs.filter((output) => output.hit_eos).length
|
||||
),
|
||||
0,
|
||||
);
|
||||
const boundaryCrossingTokens = depth.sources.reduce(
|
||||
(sum, source) => (
|
||||
sum + Object.values(source.boundary_crossing_tokens_by_condition)
|
||||
.reduce((sourceSum, value) => sourceSum + value, 0)
|
||||
),
|
||||
0,
|
||||
);
|
||||
const {
|
||||
hidden: _hiddenSummary,
|
||||
routes: _routeSummary,
|
||||
...depthHeadline
|
||||
} = depth.summary;
|
||||
|
||||
const compact = {
|
||||
schemaVersion: 1,
|
||||
artifacts: Object.fromEntries(
|
||||
Object.entries(paths)
|
||||
.filter(([name]) => name !== "output")
|
||||
.map(([name, path]) => [name, artifact(path)]),
|
||||
),
|
||||
model: {
|
||||
repo: completion.model.repo,
|
||||
revision: completion.model.revision,
|
||||
checkpointIdentity: completion.model.checkpoint_identity,
|
||||
architecture: completion.model.architecture,
|
||||
dtype: completion.model.dtype,
|
||||
},
|
||||
contract: {
|
||||
conditions,
|
||||
conditionFactors: completion.generation_contract.conditions,
|
||||
edgeOrder,
|
||||
edgePairs: depth.trace_contract.edge_pairs,
|
||||
edgeLabels,
|
||||
domains,
|
||||
targetScope: depth.source_contract.target_scope,
|
||||
decode: completion.generation_contract.decode,
|
||||
maxNewTokens: completion.generation_contract.max_new_tokens,
|
||||
useCacheGeneration: completion.generation_contract.use_cache,
|
||||
useCacheTrace: depth.trace_contract.use_cache,
|
||||
},
|
||||
completion: {
|
||||
budgetLadder: [
|
||||
{
|
||||
maxNewTokens: baseline.generation_contract.max_new_tokens,
|
||||
naturalEos: baselineNaturalEos,
|
||||
truncated: 128 - baselineNaturalEos,
|
||||
outputs: 128,
|
||||
},
|
||||
{
|
||||
maxNewTokens: 512,
|
||||
naturalEos: completionEval.summary.natural_eos,
|
||||
truncated: completionEval.summary.budget_truncated,
|
||||
outputs: completionEval.summary.outputs,
|
||||
},
|
||||
],
|
||||
summary: completionEval.summary,
|
||||
incompleteRows,
|
||||
taskRows: evaluationRows.filter(
|
||||
(row) => ["code", "math"].includes(row.domain),
|
||||
),
|
||||
taskTotals: {
|
||||
math: {
|
||||
strictCorrect: mathRows.filter(
|
||||
(row) => (
|
||||
row.taskEvaluation.strict_complete_numeric_exact
|
||||
),
|
||||
).length,
|
||||
total: mathRows.length,
|
||||
extractionMethods: Object.fromEntries(
|
||||
Object.entries(mathExtraction).map(([key, rows]) => [
|
||||
key,
|
||||
rows.length,
|
||||
]),
|
||||
),
|
||||
},
|
||||
code: {
|
||||
testsPassed: codeRows.filter(
|
||||
(row) => row.taskEvaluation.fixed_budget_tests_pass,
|
||||
).length,
|
||||
total: codeRows.length,
|
||||
statuses: Object.fromEntries(
|
||||
Object.entries(codeStatuses).map(([key, rows]) => [
|
||||
key,
|
||||
rows.length,
|
||||
]),
|
||||
),
|
||||
},
|
||||
},
|
||||
sandbox: completionEval.sandbox,
|
||||
reproduction: completionReproduction,
|
||||
execution: completion.execution,
|
||||
claimBoundary: completionEval.claim_boundary,
|
||||
},
|
||||
depth: {
|
||||
summary: depthHeadline,
|
||||
hiddenSeries,
|
||||
routerSeries,
|
||||
hiddenStages,
|
||||
routerLayers,
|
||||
boundaryCrossingTokens,
|
||||
sources: depth.sources.map((source) => ({
|
||||
id: source.id,
|
||||
domain: source.domain,
|
||||
label: source.label,
|
||||
contentTokens: source.content_tokens,
|
||||
contentTokenIdsSha256: source.content_token_ids_sha256,
|
||||
boundaryCrossingTokensByCondition: (
|
||||
source.boundary_crossing_tokens_by_condition
|
||||
),
|
||||
})),
|
||||
reproduction: depthReproduction,
|
||||
execution: depth.execution,
|
||||
claimBoundary: depth.claim_boundary,
|
||||
},
|
||||
};
|
||||
|
||||
writeFileSync(
|
||||
paths.output,
|
||||
`${JSON.stringify(compact, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
process.stdout.write(
|
||||
`${paths.output}\n`
|
||||
+ `${statSync(paths.output).size} bytes compact\n`
|
||||
+ `${completionReproduction.generatedTokenIdsExact}`
|
||||
+ ` / ${completionReproduction.cells} completion cells exact\n`
|
||||
+ `${depthReproduction.sourceObjectsExactAfterRuntimeStrip}`
|
||||
+ ` / ${depthReproduction.sources} depth sources exact`
|
||||
+ " after runtime strip\n",
|
||||
);
|
||||
@@ -80,6 +80,10 @@ const overview = await evaluate(`(() => ({
|
||||
behaviorPanels: document.querySelectorAll("[data-behavior-panel]").length,
|
||||
behaviorSources: document.querySelectorAll("[data-behavior-source] option").length,
|
||||
behaviorEdges: document.querySelectorAll("[data-behavior-map-edge]").length,
|
||||
completionDepthTabs: document.querySelectorAll("[data-cd-tab]").length,
|
||||
completionDepthPanels: document.querySelectorAll("[data-cd-panel]").length,
|
||||
hiddenStages: document.querySelectorAll("[data-hidden-stage]").length,
|
||||
routerLayers: document.querySelectorAll("[data-router-layer]").length,
|
||||
branches: document.querySelectorAll(".branch-grid > a").length,
|
||||
followups: document.querySelectorAll(".lineage-row.followup").length,
|
||||
navLinks: document.querySelectorAll(".top-nav a").length,
|
||||
@@ -879,6 +883,86 @@ await evaluate(`(() => {
|
||||
await pause(180);
|
||||
await screenshot("/tmp/llm-atlas-deepseek-behavior-desktop.png");
|
||||
|
||||
const completionDepth = await evaluate(`(() => {
|
||||
const root = document.querySelector("[data-completion-depth-lab]");
|
||||
const initial = {
|
||||
panel: root.querySelector("[data-cd-panel]:not([hidden])").dataset.cdPanel,
|
||||
conditionRows: root.querySelectorAll("[data-completion-condition]").length,
|
||||
incompleteRows: root.querySelectorAll(".incomplete-ledger article").length,
|
||||
prefixCards: root.querySelectorAll(".prefix-gate > article").length,
|
||||
};
|
||||
root.querySelector('[data-cd-tab="tasks"]').click();
|
||||
const condition = root.querySelector("[data-task-condition]");
|
||||
condition.value = "s1_eos";
|
||||
condition.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
const tasks = {
|
||||
panel: root.querySelector("[data-cd-panel]:not([hidden])").dataset.cdPanel,
|
||||
mathCards: root.querySelectorAll("[data-math-task-grid] > article").length,
|
||||
codeCards: root.querySelectorAll("[data-code-task-grid] > article").length,
|
||||
mathTotal: root.querySelector("[data-math-condition-total]").textContent.trim(),
|
||||
codeTotal: root.querySelector("[data-code-condition-total]").textContent.trim(),
|
||||
sandboxSteps: root.querySelectorAll(".sandbox-flow > article").length,
|
||||
};
|
||||
root.querySelector('[data-cd-tab="hidden"]').click();
|
||||
const hiddenEdge = root.querySelector("[data-hidden-edge]");
|
||||
const hiddenDomain = root.querySelector("[data-hidden-domain]");
|
||||
const hiddenMetric = root.querySelector("[data-hidden-metric]");
|
||||
hiddenEdge.value = "period_at_s1";
|
||||
hiddenEdge.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
hiddenDomain.value = "code";
|
||||
hiddenDomain.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
hiddenMetric.value = "relative";
|
||||
hiddenMetric.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
root.querySelector('[data-hidden-stage="8"]').click();
|
||||
const hidden = {
|
||||
panel: root.querySelector("[data-cd-panel]:not([hidden])").dataset.cdPanel,
|
||||
stages: root.querySelectorAll("[data-hidden-stage]").length,
|
||||
selected: root.querySelector("[data-hidden-stage-name]").textContent.trim(),
|
||||
cosine: root.querySelector("[data-hidden-cosine]").textContent.trim(),
|
||||
relative: root.querySelector("[data-hidden-relative]").textContent.trim(),
|
||||
exact: root.querySelector("[data-hidden-exact]").textContent.trim(),
|
||||
points: root.querySelector("[data-hidden-line]").getAttribute("points").split(" ").length,
|
||||
};
|
||||
root.querySelector('[data-cd-tab="router"]').click();
|
||||
const routerEdge = root.querySelector("[data-router-edge]");
|
||||
const routerDomain = root.querySelector("[data-router-domain]");
|
||||
routerEdge.value = "period_at_s1";
|
||||
routerEdge.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
routerDomain.value = "math";
|
||||
routerDomain.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
root.querySelector('[data-router-layer="23"]').click();
|
||||
const router = {
|
||||
panel: root.querySelector("[data-cd-panel]:not([hidden])").dataset.cdPanel,
|
||||
layers: root.querySelectorAll("[data-router-layer]").length,
|
||||
selected: root.querySelector("[data-router-layer-name]").textContent.trim(),
|
||||
ordered: root.querySelector("[data-router-ordered]").textContent.trim(),
|
||||
setExact: root.querySelector("[data-router-set]").textContent.trim(),
|
||||
tv: root.querySelector("[data-router-tv]").textContent.trim(),
|
||||
points: root.querySelector("[data-router-line]").getAttribute("points").split(" ").length,
|
||||
reproCards: root.querySelectorAll(".repro-proof > article").length,
|
||||
links: root.querySelectorAll('[data-cd-panel="router"] .artifact-links > a').length,
|
||||
};
|
||||
const first = root.querySelector('[data-cd-tab="completion"]');
|
||||
first.focus();
|
||||
first.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowRight", bubbles: true }));
|
||||
return {
|
||||
initial,
|
||||
tasks,
|
||||
hidden,
|
||||
router,
|
||||
keyboardSelected: root.querySelector('[data-cd-tab][aria-selected="true"]').dataset.cdTab,
|
||||
keyboardVisible: root.querySelector("[data-cd-panel]:not([hidden])").dataset.cdPanel,
|
||||
};
|
||||
})()`);
|
||||
await evaluate(`(() => {
|
||||
const root = document.querySelector("[data-completion-depth-lab]");
|
||||
root.querySelector('[data-cd-tab="hidden"]').click();
|
||||
root.scrollIntoView({ block: "start", behavior: "instant" });
|
||||
window.scrollBy(0, -82);
|
||||
})()`);
|
||||
await pause(180);
|
||||
await screenshot("/tmp/llm-atlas-deepseek-completion-depth-desktop.png");
|
||||
|
||||
await navigate("/");
|
||||
const home = await evaluate(`(() => ({
|
||||
releaseCards: document.querySelectorAll(".release-card").length,
|
||||
@@ -912,6 +996,7 @@ const mobile = await evaluate(`(() => {
|
||||
const root = document.querySelector("[data-deepseek-lab]");
|
||||
const artifact = document.querySelector("[data-dsv2-lab]");
|
||||
const behavior = document.querySelector("[data-behavior-lab]");
|
||||
const completionDepth = document.querySelector("[data-completion-depth-lab]");
|
||||
root.scrollIntoView({ block: "start", behavior: "instant" });
|
||||
const toggle = document.querySelector("#menu-toggle");
|
||||
toggle?.click();
|
||||
@@ -926,6 +1011,9 @@ const mobile = await evaluate(`(() => {
|
||||
behaviorSources: behavior.querySelectorAll("[data-behavior-source] option").length,
|
||||
behaviorEdges: behavior.querySelectorAll("[data-behavior-map-edge]").length,
|
||||
behaviorDeviceCells: behavior.querySelectorAll(".layer-device-map > span").length,
|
||||
completionDepthTabs: completionDepth.querySelectorAll("[data-cd-tab]").length,
|
||||
completionDepthHiddenStages: completionDepth.querySelectorAll("[data-hidden-stage]").length,
|
||||
completionDepthRouterLayers: completionDepth.querySelectorAll("[data-router-layer]").length,
|
||||
artifactHeatCells: artifact.querySelectorAll("[data-route-heatmap] > span").length,
|
||||
corpusCohorts: artifact.querySelectorAll("[data-corpus-cohort]").length,
|
||||
lengthDeltaCards: artifact.querySelectorAll("[data-length-delta-grid] > article").length,
|
||||
@@ -975,7 +1063,7 @@ const mobile = await evaluate(`(() => {
|
||||
roleBlockDomainCards: artifact.querySelectorAll("[data-role-block-domain-grid] > article").length,
|
||||
roleBlockDepthCells: artifact.querySelectorAll("[data-role-block-depth-map] > div > span").length,
|
||||
offenders: [...document.querySelectorAll("body *")]
|
||||
.filter((node) => !node.closest(".paper-chain, .advantage-table, .precision-table, .mapping-table, [data-deepseek-lab], [data-dsv2-lab], [data-behavior-lab]"))
|
||||
.filter((node) => !node.closest(".paper-chain, .advantage-table, .precision-table, .mapping-table, [data-deepseek-lab], [data-dsv2-lab], [data-behavior-lab], [data-completion-depth-lab]"))
|
||||
.filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1)
|
||||
.slice(0, 12)
|
||||
.map((node) => ({
|
||||
@@ -1107,20 +1195,29 @@ await evaluate(`(() => {
|
||||
})()`);
|
||||
await pause(180);
|
||||
await screenshot("/tmp/llm-atlas-deepseek-behavior-mobile.png");
|
||||
await evaluate(`(() => {
|
||||
const completionDepth = document.querySelector("[data-completion-depth-lab]");
|
||||
completionDepth.querySelector('[data-cd-tab="router"]').click();
|
||||
completionDepth.scrollIntoView({ block: "start", behavior: "instant" });
|
||||
window.scrollBy(0, -70);
|
||||
})()`);
|
||||
await pause(180);
|
||||
await screenshot("/tmp/llm-atlas-deepseek-completion-depth-mobile.png");
|
||||
|
||||
const report = { overview, capacity, cache, codesign, rl, artifactRoute, artifactLoad, artifactCache, artifactAbsorb, artifactCorpus, artifactTemplate, artifactHistory, artifactDistance, artifactBoundary, artifactRole, artifactSpecial, artifactRoleBlock, artifactEvidence, behavior, home, papers, mobile, exceptions };
|
||||
const report = { overview, capacity, cache, codesign, rl, artifactRoute, artifactLoad, artifactCache, artifactAbsorb, artifactCorpus, artifactTemplate, artifactHistory, artifactDistance, artifactBoundary, artifactRole, artifactSpecial, artifactRoleBlock, artifactEvidence, behavior, completionDepth, home, papers, mobile, exceptions };
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
|
||||
const numeric = (text) => Number.parseFloat(text.replaceAll(",", ""));
|
||||
const failures = [];
|
||||
if (!overview.title.includes("为什么转向")) failures.push("专题标题异常");
|
||||
if (overview.sections !== 27 || overview.tocLinks !== 27) failures.push("二十六个编号专题加阅读链的目录结构异常");
|
||||
if (overview.sections !== 28 || overview.tocLinks !== 28) failures.push("二十七个编号专题加阅读链的目录结构异常");
|
||||
if (overview.ledgers !== 24 || overview.waves !== 10) failures.push("二十四张问题账或十次转向结构异常");
|
||||
if (overview.paperLinks !== 60 || overview.branches !== 5 || overview.followups !== 1) failures.push("论文链、旁支或公开后续标记异常");
|
||||
if (overview.labTabs !== 4 || overview.labPanels !== 4) failures.push("四联实验结构异常");
|
||||
if (overview.artifactTabs !== 13 || overview.artifactPanels !== 13 || overview.artifactLayers !== 27) failures.push("真实权重十三联实验结构异常");
|
||||
if (overview.behaviorTabs !== 4 || overview.behaviorPanels !== 4 || overview.behaviorSources !== 16 || overview.behaviorEdges !== 10) failures.push("Chat 行为实验结构异常");
|
||||
if (overview.heroLabs !== "18 个可操作实验") failures.push("DeepSeek 实验总数账异常");
|
||||
if (overview.completionDepthTabs !== 4 || overview.completionDepthPanels !== 4 || overview.hiddenStages !== 29 || overview.routerLayers !== 26) failures.push("Chat 完成度与全深度实验结构异常");
|
||||
if (overview.heroLabs !== "19 个可操作实验") failures.push("DeepSeek 实验总数账异常");
|
||||
if (overview.navLinks !== 20 || home.navLinks !== 20 || mobile.mobileLinks !== 20 || overview.activeNav !== "DeepSeek") failures.push("全站导航未同步 DeepSeek");
|
||||
if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在文档级横向溢出");
|
||||
if (capacity.initial.panel !== "capacity" || capacity.initial.total !== "32.1× FFN" || capacity.initial.active !== "1.13× FFN") failures.push("V3 稀疏容量初始账异常");
|
||||
@@ -1213,9 +1310,14 @@ if (behavior.mathMap.panel !== "map" || behavior.mathMap.rows !== 10 || !behavio
|
||||
if (behavior.execution.panel !== "execution" || behavior.execution.layers !== 29 || behavior.execution.gpu !== 25 || behavior.execution.cpu !== 4 || behavior.execution.runtimeCards !== 4) failures.push("Chat BF16 GPU / CPU offload 设备图异常");
|
||||
if (behavior.boundary.panel !== "boundary" || behavior.boundary.ladder !== 3 || behavior.boundary.tokenCards !== 4 || behavior.boundary.reproCards !== 4 || behavior.boundary.links !== 3 || !behavior.boundary.forbidden.includes("route TV")) failures.push("Chat 行为证据阶梯或限制异常");
|
||||
if (behavior.keyboardSelected !== "map" || behavior.keyboardVisible !== "map") failures.push("Chat 行为实验键盘 tab 导航异常");
|
||||
if (completionDepth.initial.panel !== "completion" || completionDepth.initial.conditionRows !== 8 || completionDepth.initial.incompleteRows !== 7 || completionDepth.initial.prefixCards !== 3) failures.push("512-token 完成度阶梯结构异常");
|
||||
if (completionDepth.tasks.panel !== "tasks" || completionDepth.tasks.mathCards !== 4 || completionDepth.tasks.codeCards !== 4 || completionDepth.tasks.mathTotal !== "3 / 4 PASS" || completionDepth.tasks.codeTotal !== "2 / 4 PASS" || completionDepth.tasks.sandboxSteps !== 4) failures.push("Math / HumanEval 完成感知评测切换异常");
|
||||
if (completionDepth.hidden.panel !== "hidden" || completionDepth.hidden.stages !== 29 || completionDepth.hidden.selected !== "layer_07" || completionDepth.hidden.points !== 29 || completionDepth.hidden.exact === "1,537 / 1,537" || numeric(completionDepth.hidden.relative) <= 0) failures.push("29 阶段隐藏状态曲线或交互异常");
|
||||
if (completionDepth.router.panel !== "router" || completionDepth.router.layers !== 26 || completionDepth.router.selected !== "layer 24" || completionDepth.router.points !== 26 || !completionDepth.router.ordered.includes("%") || !completionDepth.router.setExact.includes("%") || numeric(completionDepth.router.tv) <= 0 || completionDepth.router.reproCards !== 4 || completionDepth.router.links !== 4) failures.push("26 层 MoE 路由曲线或复跑证据异常");
|
||||
if (completionDepth.keyboardSelected !== "tasks" || completionDepth.keyboardVisible !== "tasks") failures.push("完成度与全深度实验键盘 tab 导航异常");
|
||||
if (home.releaseCards !== 17 || !home.firstRelease.includes("47 页不再压成摘要") || home.firstHref !== "/k3/" || home.paperCount !== "486") failures.push("首页 DeepSeek 首发入口或论文数异常");
|
||||
if (papers.total !== 486 || !papers.hasFilter || papers.visible < 20 || !papers.hasCoder || !papers.hasEngram) failures.push("论文库 DeepSeek 聚光异常");
|
||||
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4 || mobile.artifactTabs !== 13 || mobile.behaviorTabs !== 4 || mobile.behaviorSources !== 16 || mobile.behaviorEdges !== 10 || mobile.behaviorDeviceCells !== 29 || mobile.artifactHeatCells !== 64 || mobile.corpusCohorts !== 3 || mobile.lengthDeltaCards !== 4 || mobile.templateLayers !== 6 || mobile.templateScopes !== 2 || mobile.templateModes !== 2 || mobile.templateDomainCards !== 4 || mobile.templateDepthCells !== 24 || mobile.historyLayers !== 6 || mobile.historyScopes !== 2 || mobile.historyModes !== 2 || mobile.historyEffects !== 3 || mobile.historyDomainCards !== 4 || mobile.historyDepthCells !== 24 || mobile.distanceLayers !== 6 || mobile.distanceScopes !== 2 || mobile.distanceModes !== 2 || mobile.distanceContrasts !== 2 || mobile.distanceDomainCards !== 4 || mobile.distanceDepthCells !== 24 || mobile.boundaryLayers !== 6 || mobile.boundaryScopes !== 2 || mobile.boundaryModes !== 2 || mobile.boundaryContrasts !== 3 || mobile.boundaryTokenCards !== 4 || mobile.boundaryDomainCards !== 4 || mobile.boundaryDepthCells !== 24 || mobile.roleLayers !== 6 || mobile.roleScopes !== 2 || mobile.roleModes !== 2 || mobile.roleContrasts !== 3 || mobile.roleLevelCards !== 4 || mobile.roleDomainCards !== 4 || mobile.roleDepthCells !== 24 || mobile.specialLayers !== 6 || mobile.specialScopes !== 2 || mobile.specialModes !== 2 || mobile.specialContrasts !== 4 || mobile.specialTokenCards !== 4 || mobile.specialDomainCards !== 4 || mobile.specialDepthCells !== 24 || mobile.roleBlockLayers !== 6 || mobile.roleBlockScopes !== 2 || mobile.roleBlockModes !== 2 || mobile.roleBlockEffects !== 3 || mobile.roleBlockMatrixCards !== 4 || mobile.roleBlockDomainCards !== 4 || mobile.roleBlockDepthCells !== 24) failures.push("移动端导航或实验异常");
|
||||
if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4 || mobile.artifactTabs !== 13 || mobile.behaviorTabs !== 4 || mobile.behaviorSources !== 16 || mobile.behaviorEdges !== 10 || mobile.behaviorDeviceCells !== 29 || mobile.completionDepthTabs !== 4 || mobile.completionDepthHiddenStages !== 29 || mobile.completionDepthRouterLayers !== 26 || mobile.artifactHeatCells !== 64 || mobile.corpusCohorts !== 3 || mobile.lengthDeltaCards !== 4 || mobile.templateLayers !== 6 || mobile.templateScopes !== 2 || mobile.templateModes !== 2 || mobile.templateDomainCards !== 4 || mobile.templateDepthCells !== 24 || mobile.historyLayers !== 6 || mobile.historyScopes !== 2 || mobile.historyModes !== 2 || mobile.historyEffects !== 3 || mobile.historyDomainCards !== 4 || mobile.historyDepthCells !== 24 || mobile.distanceLayers !== 6 || mobile.distanceScopes !== 2 || mobile.distanceModes !== 2 || mobile.distanceContrasts !== 2 || mobile.distanceDomainCards !== 4 || mobile.distanceDepthCells !== 24 || mobile.boundaryLayers !== 6 || mobile.boundaryScopes !== 2 || mobile.boundaryModes !== 2 || mobile.boundaryContrasts !== 3 || mobile.boundaryTokenCards !== 4 || mobile.boundaryDomainCards !== 4 || mobile.boundaryDepthCells !== 24 || mobile.roleLayers !== 6 || mobile.roleScopes !== 2 || mobile.roleModes !== 2 || mobile.roleContrasts !== 3 || mobile.roleLevelCards !== 4 || mobile.roleDomainCards !== 4 || mobile.roleDepthCells !== 24 || mobile.specialLayers !== 6 || mobile.specialScopes !== 2 || mobile.specialModes !== 2 || mobile.specialContrasts !== 4 || mobile.specialTokenCards !== 4 || mobile.specialDomainCards !== 4 || mobile.specialDepthCells !== 24 || mobile.roleBlockLayers !== 6 || mobile.roleBlockScopes !== 2 || mobile.roleBlockModes !== 2 || mobile.roleBlockEffects !== 3 || mobile.roleBlockMatrixCards !== 4 || mobile.roleBlockDomainCards !== 4 || mobile.roleBlockDepthCells !== 24) failures.push("移动端导航或实验异常");
|
||||
if (mobile.offenders.length) failures.push(`移动端越界元素：${JSON.stringify(mobile.offenders)}`);
|
||||
if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user