--- import rawTrace from "@/data/deepseek-v2-lite-trace.json"; import rawRepro from "@/data/deepseek-v2-lite-trace-repro.json"; import rawAbsorb from "@/data/deepseek-v2-lite-absorb.json"; import rawAbsorbRepro from "@/data/deepseek-v2-lite-absorb-repro.json"; import rawCorpus from "@/data/deepseek-v2-lite-routing-corpus.json"; import rawCorpusRepro from "@/data/deepseek-v2-lite-routing-corpus-repro.json"; import rawMatched16 from "@/data/deepseek-v2-lite-routing-matched16.json"; import rawMatched16Repro from "@/data/deepseek-v2-lite-routing-matched16-repro.json"; import rawMatched24 from "@/data/deepseek-v2-lite-routing-matched24.json"; import rawMatched24Repro from "@/data/deepseek-v2-lite-routing-matched24-repro.json"; import rawLengthSensitivity from "@/data/deepseek-v2-lite-routing-length-sensitivity.json"; import rawTemplate from "@/data/deepseek-v2-lite-routing-template.json"; import rawTemplateRepro from "@/data/deepseek-v2-lite-routing-template-repro.json"; import rawHistory from "@/data/deepseek-v2-lite-routing-history-factorial.json"; import rawHistoryRepro from "@/data/deepseek-v2-lite-routing-history-factorial-repro.json"; import rawDistance from "@/data/deepseek-v2-lite-routing-history-distance-control.json"; import rawDistanceRepro from "@/data/deepseek-v2-lite-routing-history-distance-control-repro.json"; import rawBoundaryCompact from "@/data/deepseek-v2-lite-routing-history-boundary-token-control-compact.json"; import rawRoleCompact from "@/data/deepseek-v2-lite-routing-role-marker-head-control-compact.json"; import rawSpecialCompact from "@/data/deepseek-v2-lite-routing-special-token-family-control-compact.json"; import rawRoleBlockCompact from "@/data/deepseek-v2-lite-routing-role-marker-block-factorial-compact.json"; const trace = rawTrace as any; const repro = rawRepro as any; const absorb = rawAbsorb as any; const absorbRepro = rawAbsorbRepro as any; const corpus = rawCorpus as any; const corpusRepro = rawCorpusRepro as any; const matched16 = rawMatched16 as any; const matched16Repro = rawMatched16Repro as any; const matched24 = rawMatched24 as any; const matched24Repro = rawMatched24Repro as any; const lengthSensitivity = rawLengthSensitivity as any; const template = rawTemplate as any; const templateRepro = rawTemplateRepro as any; const history = rawHistory as any; const historyRepro = rawHistoryRepro as any; const distance = rawDistance as any; const distanceRepro = rawDistanceRepro as any; const boundaryCompact = rawBoundaryCompact as any; const roleCompact = rawRoleCompact as any; const specialCompact = rawSpecialCompact as any; const roleBlockCompact = rawRoleBlockCompact as any; const absorbExact = JSON.stringify(absorb) === JSON.stringify(absorbRepro); const corpusExact = JSON.stringify(corpus) === JSON.stringify(corpusRepro); const matched16Exact = JSON.stringify(matched16) === JSON.stringify(matched16Repro); const matched24Exact = JSON.stringify(matched24) === JSON.stringify(matched24Repro); const templateExact = JSON.stringify(template) === JSON.stringify(templateRepro); const historyExact = JSON.stringify(history) === JSON.stringify(historyRepro); const distanceExact = JSON.stringify(distance) === JSON.stringify(distanceRepro); const boundaryExact = boundaryCompact.source.exact; const roleExact = roleCompact.source.exact; const specialExact = specialCompact.source.exact; const roleBlockExact = roleBlockCompact.source.exact; const bytes = (value: number) => value >= 1024 ? `${(value / 1024).toFixed(2)} KiB` : `${value.toLocaleString()} B`; const scientific = (value: number) => value.toExponential(2); const compact = { prompts: trace.prompts.map((prompt: any) => ({ id: prompt.id, label: prompt.label, text: prompt.text, })), layers: trace.execution.layers.slice(1).map((layer: any) => ({ layer: layer.layer, load: layer.routing.aggregate_load, metrics: layer.routing.aggregate_metrics, selectedWeight: layer.routing.selected_weight_sum, topExperts: layer.routing.top_experts, prompts: layer.routing.per_prompt.map((prompt: any) => ({ id: prompt.id, label: prompt.label, tokens: prompt.tokens, load: prompt.load, metrics: prompt.metrics, topExperts: prompt.top_experts, tokenRoutes: prompt.token_routes, })), jaccard: layer.routing.prompt_pair_jaccard, })), cache: trace.cache_accounting, execution: { batch: trace.execution.batch, paddedSequence: trace.execution.padded_sequence, validTokens: trace.execution.valid_tokens, }, }; const compactJson = JSON.stringify(compact).replaceAll("<", "\\u003c"); const compactCorpus = (input: any) => ({ domains: input.corpus_contract.domains, labels: input.corpus_contract.domain_labels, counts: input.corpus_contract.counts, inference: input.inference_contract, statistics: input.statistical_contract, layers: input.layers.slice(1).map((layer: any) => ({ layer: layer.layer, routes: layer.routes, modes: layer.statistics, })), }); const corpusCompact = { cohorts: { natural: compactCorpus(corpus), matched16: compactCorpus(matched16), matched24: compactCorpus(matched24), }, lengthSensitivity, }; const corpusCompactJson = JSON.stringify(corpusCompact).replaceAll("<", "\\u003c"); const aggregateTemplateAlignment = (layer: any, domain: string) => { const rows = layer.prompts .filter((prompt: any) => prompt.domain === domain) .map((prompt: any) => prompt.alignments.raw_to_user_content); const aligned = rows.reduce((sum: number, row: any) => sum + row.aligned_tokens, 0); const setExact = rows.reduce((sum: number, row: any) => sum + row.set_topk_exact, 0); const orderedExact = rows.reduce((sum: number, row: any) => sum + row.ordered_topk_exact, 0); const weightedJaccard = rows.reduce( (sum: number, row: any) => sum + row.mean_jaccard * row.aligned_tokens, 0, ); return { aligned, setExactRate: setExact / aligned, orderedExactRate: orderedExact / aligned, meanJaccard: weightedJaccard / aligned, }; }; const templateCompact = { domains: template.corpus_contract.domains, labels: template.corpus_contract.domain_labels, inference: template.inference_contract, template: { sha256: template.template_contract.chat_template_sha256, bos: template.template_contract.bos_token_id, }, exact: templateExact, layers: template.layers.slice(1).map((layer: any) => ({ layer: layer.layer, invariant: layer.causal_suffix_invariant, alignment: Object.fromEntries( template.corpus_contract.domains.map((domain: string) => [ domain, aggregateTemplateAlignment(layer, domain), ]), ), scopes: Object.fromEntries( ["content_only", "full_input"].map((scope) => [ scope, { modes: Object.fromEntries( ["prompt_balanced", "token_weighted"].map((mode) => [ mode, { rawToUser: layer.statistics[scope].modes[mode] .comparisons.raw_to_user, userToGeneration: layer.statistics[scope].modes[mode] .comparisons.user_to_generation, }, ]), ), }, ]), ), })), }; const templateCompactJson = JSON.stringify(templateCompact).replaceAll("<", "\\u003c"); const historyConditions = ["s0f0", "s1f0", "s0f1", "s1f1"]; const historyEdges = ["system_at_f0", "system_at_f1", "fewshot_at_s0", "fewshot_at_s1"]; const aggregateHistoryAlignment = (layer: any, domain: string, edge: string) => { const rows = layer.prompts .filter((prompt: any) => prompt.domain === domain) .map((prompt: any) => prompt.alignments[edge]); const aligned = rows.reduce((sum: number, row: any) => sum + row.aligned_tokens, 0); const setExact = rows.reduce((sum: number, row: any) => sum + row.set_topk_exact, 0); const orderedExact = rows.reduce((sum: number, row: any) => sum + row.ordered_topk_exact, 0); const weightedJaccard = rows.reduce( (sum: number, row: any) => sum + row.mean_jaccard * row.aligned_tokens, 0, ); return { aligned, setExactRate: setExact / aligned, orderedExactRate: orderedExact / aligned, meanJaccard: weightedJaccard / aligned, }; }; const historyCompact = { domains: history.corpus_contract.domains, labels: history.corpus_contract.domain_labels, inference: history.inference_contract, messages: { system: history.message_history_contract.system_message, demoUser: history.message_history_contract.demo_user, demoAssistant: history.message_history_contract.demo_assistant, }, exact: historyExact, layers: history.layers.slice(1).map((layer: any) => ({ layer: layer.layer, alignment: Object.fromEntries( history.corpus_contract.domains.map((domain: string) => [ domain, Object.fromEntries( historyEdges.map((edge) => [ edge, aggregateHistoryAlignment(layer, domain, edge), ]), ), ]), ), scopes: Object.fromEntries( ["target_content", "full_input"].map((scope) => [ scope, { modes: Object.fromEntries( ["prompt_balanced", "token_weighted"].map((mode) => { const statistics = layer.statistics[scope].modes[mode]; return [ mode, { conditions: Object.fromEntries( historyConditions.map((condition) => [ condition, Object.fromEntries( history.corpus_contract.domains.map((domain: string) => [ domain, statistics.conditions[condition][domain].metrics.cv.point, ]), ), ]), ), factorial: Object.fromEntries( history.corpus_contract.domains.map((domain: string) => [ domain, statistics.factorial[domain].metric_effects.cv, ]), ), comparisons: Object.fromEntries( historyEdges.map((edge) => [ edge, Object.fromEntries( history.corpus_contract.domains.map((domain: string) => [ domain, { cv: statistics.comparisons[edge][domain].metrics.cv, tv: statistics.comparisons[edge][domain].total_variation, jsd: statistics.comparisons[edge][domain].js_divergence, }, ]), ), ]), ), }, ]; }), ), }, ]), ), })), }; const historyCompactJson = JSON.stringify(historyCompact).replaceAll("<", "\\u003c"); const distanceEdges = [ "system_none", "system_filler", "system_demo", "demo_vs_filler_s0", "demo_vs_filler_s1", ]; const aggregateDistanceAlignment = (layer: any, domain: string, edge: string) => { const rows = layer.prompts .filter((prompt: any) => prompt.domain === domain) .map((prompt: any) => prompt.alignments[edge]); const aligned = rows.reduce((sum: number, row: any) => sum + row.aligned_tokens, 0); const setExact = rows.reduce((sum: number, row: any) => sum + row.set_topk_exact, 0); const orderedExact = rows.reduce((sum: number, row: any) => sum + row.ordered_topk_exact, 0); const weightedJaccard = rows.reduce( (sum: number, row: any) => sum + row.mean_jaccard * row.aligned_tokens, 0, ); return { aligned, setExactRate: setExact / aligned, orderedExactRate: orderedExact / aligned, meanJaccard: weightedJaccard / aligned, }; }; const distanceCompact = { domains: distance.corpus_contract.domains, labels: distance.corpus_contract.domain_labels, inference: distance.inference_contract, messages: { system: distance.history_control_contract.system_message, demoUser: distance.history_control_contract.demo_user, demoAssistant: distance.history_control_contract.demo_assistant, fillerUser: distance.history_control_contract.filler_user, fillerAssistant: distance.history_control_contract.filler_assistant, }, exact: distanceExact, layers: distance.layers.slice(1).map((layer: any) => ({ layer: layer.layer, alignment: Object.fromEntries( distance.corpus_contract.domains.map((domain: string) => [ domain, Object.fromEntries( distanceEdges.map((edge) => [ edge, aggregateDistanceAlignment(layer, domain, edge), ]), ), ]), ), scopes: Object.fromEntries( ["target_content", "full_input"].map((scope) => [ scope, { modes: Object.fromEntries( ["prompt_balanced", "token_weighted"].map((mode) => { const control = layer.statistics[scope].modes[mode].history_control; return [ mode, Object.fromEntries( distance.corpus_contract.domains.map((domain: string) => [ domain, { distances: control[domain].system_edge_distances, contrasts: control[domain].system_edge_distance_contrasts, cvEdges: control[domain].metric_system_edges.cv, cvContrasts: control[domain].metric_system_edge_contrasts.cv, lexical: control[domain].lexical_replacement, }, ]), ), ]; }), ), }, ]), ), })), }; const distanceCompactJson = JSON.stringify(distanceCompact).replaceAll("<", "\\u003c"); const boundaryCompactJson = JSON.stringify(boundaryCompact).replaceAll("<", "\\u003c"); const roleCompactJson = JSON.stringify(roleCompact).replaceAll("<", "\\u003c"); const specialCompactJson = JSON.stringify(specialCompact).replaceAll("<", "\\u003c"); const roleBlockCompactJson = JSON.stringify(roleBlockCompact).replaceAll("<", "\\u003c"); const aggregateCrossBatch = (input: any) => [1, 2, 3, 4, 5, 6].map((layer) => { const rows = input.crossBatch.filter((row: any) => row.layer === layer); return { layer, compared: rows.reduce((sum: number, row: any) => sum + row.compared, 0), targetRouteHashExact: rows.reduce( (sum: number, row: any) => sum + row.targetRouteHashExact, 0, ), targetLoadExact: rows.reduce( (sum: number, row: any) => sum + row.targetLoadExact, 0, ), }; }); const specialCrossBatch = aggregateCrossBatch(specialCompact); const roleBlockCrossBatch = aggregateCrossBatch(roleBlockCompact); const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoint_tensor_bytes; ---

REAL-WEIGHT LAB / DEEPSEEK-V2-LITE

这一次不是调公式:让官方权重真的走过七层

固定官方 revision、tokenizer、模型代码和 BF16 第一分片;RTX 5090 连续执行 layer 0–6, 从 3,240 次 token 显微轨迹扩到 11,289,744 次公开语料路由,并让 layer-1 权重继续走入官方吸收式 cache。 所有结论都带证据身份与停止线。

X本机执行真实权重与 hidden states O官方工件配置、代码与 checkpoint D确定推导从 shape 计算 bytes U未覆盖全模型、训练与生产 kernel
X / TOKEN-LEVEL ROUTES

先选层,再选 prompt 和 token

同一个 expert ID 只在当前层内有意义;跨层同号专家是不同参数,图中不会把它们连成“专长轨迹”。

PROMPT

TOKEN #0 ID
ACTUAL TOP-6 / WEIGHT DESCENDING
SELECTED WEIGHT SUM 配置不把 top-6 重新归一到 1
64 ROUTED EXPERTS / CURRENT PROMPT

颜色表示这条 prompt 在当前层的选择次数;空白表示本小样本未触达,不表示专家失效。

ROUTES

tokens × top-6

USED / 64

至少被选一次

CV

std ÷ mean

EFFECTIVE

exp(route entropy)

X / DESCRIPTIVE TRACE

这是 4 条固定 prompt、90 个有效 token 的前六个 MoE 层;不能据此命名专家、估计线上总体负载或判断训练均衡。

可复现入口 experiments/deepseek/v2_lite_trace.py · experiments/deepseek/v2_lite_absorb_probe.py · experiments/deepseek/v2_lite_routing_corpus.py · experiments/deepseek/compare_routing_length_control.py · research/DEEPSEEK_ROUTING_LENGTH_CONTROL_AUDIT.md · experiments/deepseek/v2_lite_routing_template_probe.py · research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md · experiments/deepseek/v2_lite_routing_history_factorial_probe.py · research/DEEPSEEK_ROUTING_HISTORY_FACTORIAL_AUDIT.md · experiments/deepseek/v2_lite_routing_history_distance_control.py · research/DEEPSEEK_ROUTING_HISTORY_DISTANCE_CONTROL_AUDIT.md · experiments/deepseek/v2_lite_routing_history_boundary_token_control.py · research/DEEPSEEK_ROUTING_HISTORY_BOUNDARY_TOKEN_AUDIT.md · experiments/deepseek/v2_lite_routing_role_marker_head_control.py · research/DEEPSEEK_ROUTING_ROLE_MARKER_HEAD_AUDIT.md · experiments/deepseek/v2_lite_routing_special_token_family_control.py · research/DEEPSEEK_ROUTING_SPECIAL_TOKEN_FAMILY_AUDIT.md · experiments/deepseek/v2_lite_routing_role_marker_block_factorial.py · research/DEEPSEEK_ROUTING_ROLE_MARKER_BLOCK_AUDIT.md