feat: map DeepSeek Chat sampling robustness
This commit is contained in:
wuyang
@@ -0,0 +1,998 @@
|
||||
---
|
||||
import rawLab from "@/data/deepseek-v2-lite-chat-sampling-compact.json";
|
||||
|
||||
const lab = rawLab as any;
|
||||
const json = JSON.stringify(lab).replaceAll("<", "\\u003c");
|
||||
const conditions = lab.contract.conditions as string[];
|
||||
const edges = lab.contract.edges as string[];
|
||||
const conditionLabels: Record<string, string> = {
|
||||
s0_eos: "S0 · EOS",
|
||||
s1_eos: "S1 · EOS",
|
||||
s0_bos: "S0 · BOS",
|
||||
s1_bos: "S1 · BOS",
|
||||
s0_x: "S0 · x",
|
||||
s1_x: "S1 · x",
|
||||
s0_period: "S0 · 句点",
|
||||
s1_period: "S1 · 句点",
|
||||
};
|
||||
const sourceLabels: Record<string, string> = {
|
||||
"wikitext2/raw-validation/0443": "English · WikiText",
|
||||
"tnews/test/4855": "中文 · TNEWS",
|
||||
"HumanEval/31": "Code · HumanEval/31",
|
||||
"gsm8k/test/1069": "Math · GSM8K/1069",
|
||||
};
|
||||
---
|
||||
|
||||
<figure class="sampling-lab" data-sampling-lab>
|
||||
<header class="sp-head">
|
||||
<div>
|
||||
<p>ROUND 06 / PREREGISTERED SAMPLING</p>
|
||||
<h3>greedy 只是一条路：固定八个 seed，打开有限的生成轨迹集合</h3>
|
||||
</div>
|
||||
<p>
|
||||
同一官方 SFT Chat checkpoint、同四条 source、同八格 prompt batch。
|
||||
只把解码切到官方 <code>temperature=.3 · top_p=.95</code>，显式关闭 top-k；
|
||||
生成、任务评测与新进程复跑仍分三层保存。
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<div class="sp-ledger">
|
||||
<article class="pass"><span>SAMPLED OUTPUTS</span><b>256</b><p>4 sources × 8 seeds × 8 cells</p></article>
|
||||
<article class="pass"><span>NATURAL EOS</span><b>251 / 256</b><p>5 格在 512-token 触顶</p></article>
|
||||
<article><span>UNIQUE TRAJECTORIES</span><b>242 / 256</b><p>完整 token hash；不是语义类别</p></article>
|
||||
<article><span>MATH · STRICT</span><b>62 / 64</b><p>同一题的重复采样</p></article>
|
||||
<article><span>CODE · TESTS</span><b>63 / 64</b><p>24 个唯一执行候选</p></article>
|
||||
<article class="pass"><span>FRESH PROCESS</span><b>64 / 64</b><p>八项预注册字段 exact</p></article>
|
||||
</div>
|
||||
|
||||
<div class="sp-tabs" role="tablist" aria-label="选择采样稳健性实验视图">
|
||||
<button type="button" role="tab" data-sp-tab="trajectories" aria-selected="true">
|
||||
<span>01</span><b>八个 seed 生成了什么</b><small>trajectory microscope</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-sp-tab="tasks" aria-selected="false" tabindex="-1">
|
||||
<span>02</span><b>完成为什么仍会答错</b><small>math + code ledgers</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-sp-tab="edges" aria-selected="false" tabindex="-1">
|
||||
<span>03</span><b>两个样本集合怎样比</b><small>aligned + nearest set</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-sp-tab="reproduction" aria-selected="false" tabindex="-1">
|
||||
<span>04</span><b>随机但仍可复现</b><small>seed contract + rerun</small>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<section class="sp-panel" data-sp-panel="trajectories">
|
||||
<div class="sp-panel-lead">
|
||||
<div><span>I / TRAJECTORY MICROSCOPE</span><h4>同一格不是一个答案，而是八条有限样本</h4></div>
|
||||
<p>
|
||||
点任意 seed 查看长度、停止状态、hash 与短预览。unique 只比较完整 token IDs；
|
||||
pairwise similarity 用 token 编辑距离，不把同义改写冒充 exact。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="sp-controls">
|
||||
<label>
|
||||
<span>SOURCE</span>
|
||||
<select data-sp-source aria-label="选择采样 source">
|
||||
{lab.sources.map((source: any) => (
|
||||
<option value={source.id}>{sourceLabels[source.id]}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
<span>CONDITION</span>
|
||||
<select data-sp-condition aria-label="选择采样 condition">
|
||||
{conditions.map((condition) => (
|
||||
<option value={condition}>{conditionLabels[condition]}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<div class="sampling-config">
|
||||
<span>DECODE CONTRACT</span>
|
||||
<b>T .3 · P .95 · K 0 · CAP 512</b>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="trajectory-summary">
|
||||
<article><span>UNIQUE / 8</span><b data-sp-unique>—</b><p>完整 trajectory hash</p></article>
|
||||
<article><span>NATURAL EOS / 8</span><b data-sp-eos>—</b><p>触顶与自然结束分开</p></article>
|
||||
<article><span>GREEDY IN SET</span><b data-sp-greedy>—</b><p>八样本是否抽到 mode 轨迹</p></article>
|
||||
<article><span>PAIRWISE SIM</span><b data-sp-pairwise>—</b><p>28 对 mean · min–max</p></article>
|
||||
</div>
|
||||
|
||||
<div class="seed-stage">
|
||||
<header>
|
||||
<div><span>8 PREREGISTERED SEEDS</span><b data-sp-stage-title>—</b></div>
|
||||
<p><i class="eos"></i>EOS <i class="truncated"></i>512 截断 <i class="repeat"></i>重复完整轨迹</p>
|
||||
</header>
|
||||
<div class="seed-bars" data-sp-seeds></div>
|
||||
<article class="seed-focus" data-sp-focus>
|
||||
<div>
|
||||
<span data-sp-focus-seed>选择一个 seed</span>
|
||||
<b data-sp-focus-meta>—</b>
|
||||
</div>
|
||||
<p data-sp-focus-preview>—</p>
|
||||
<code data-sp-focus-hash>—</code>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="condition-overview" aria-label="八个 condition 的采样摘要">
|
||||
<header><b>CONDITION</b><b>EOS</b><b>MEAN TOKENS</b><b>UNIQUE</b><b>GREEDY SETS</b></header>
|
||||
{conditions.map((condition) => {
|
||||
const row = lab.conditions[condition];
|
||||
return (
|
||||
<div>
|
||||
<span>{conditionLabels[condition]}</span>
|
||||
<b class={row.naturalEos === 32 ? "good" : "warn"}>{row.naturalEos} / 32</b>
|
||||
<b>{row.meanGeneratedTokens.toFixed(1)}</b>
|
||||
<b>{row.uniqueTrajectoriesAcrossSourceSets} / 32</b>
|
||||
<b>{row.greedyIncludedSourceSets} / 4</b>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<aside class="sp-warning">
|
||||
<b>9 / 32 个八样本集合包含 greedy 完整轨迹</b>
|
||||
<p>
|
||||
这不等于 greedy “不可信”。greedy 每步选 mode；nucleus sampling 从截断后的分布抽样。
|
||||
只抽八次没有遇到 mode 路径很正常。
|
||||
</p>
|
||||
</aside>
|
||||
</section>
|
||||
|
||||
<section class="sp-panel" data-sp-panel="tasks" hidden>
|
||||
<div class="sp-panel-lead">
|
||||
<div><span>II / TASK LEDGERS</span><h4>自然结束、能解析、能运行，仍然不保证正确</h4></div>
|
||||
<p>
|
||||
这里各只有一条 task source。64 是 8 conditions × 8 seeds，不是 64 道题；
|
||||
gold 与 tests 在生成冻结后才进入独立 evaluator。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="task-condition-control">
|
||||
<label><span>查看一个 condition 的 8 个 seed</span>
|
||||
<select data-sp-task-condition aria-label="选择任务 condition">
|
||||
{conditions.map((condition) => (
|
||||
<option value={condition}>{conditionLabels[condition]}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<div><span>全局任务账</span><b>Math 62 / 64 · Code 63 / 64</b></div>
|
||||
</div>
|
||||
|
||||
<div class="sample-task-grid">
|
||||
<article>
|
||||
<header><div><span>GSM8K / 1069</span><b data-sp-math-total>—</b></div><em>gold = 300</em></header>
|
||||
<div class="seed-task-cells" data-sp-math-cells></div>
|
||||
</article>
|
||||
<article>
|
||||
<header><div><span>HUMANEVAL / 31</span><b data-sp-code-total>—</b></div><em>official tests</em></header>
|
||||
<div class="seed-task-cells" data-sp-code-cells></div>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="failure-cases">
|
||||
<article>
|
||||
<span>MATH FAILURE · 2 / 64</span>
|
||||
<h5>“坏掉 1/4”被误写成“只剩 1/4”</h5>
|
||||
<div class="reasoning-bug">
|
||||
<b>20 trucks × 1/4 = 5</b><i>错误语义跳跃</i><b>5 × 20 = 100</b>
|
||||
</div>
|
||||
<p>
|
||||
两条都自然 EOS，并明确输出 <code>{"\\boxed{100}"}</code>；抽取器没有错，
|
||||
是推理把仍可工作的 <code>3/4</code> 丢掉了。
|
||||
</p>
|
||||
</article>
|
||||
<article>
|
||||
<span>CODE FAILURE · 1 / 64</span>
|
||||
<h5>AST 合法、可以执行，却漏掉所有大于 2 的偶数</h5>
|
||||
<pre><code>if n < 2: return False
|
||||
if n == 2: return True
|
||||
for i in range(3, sqrt(n), 2): ...
|
||||
# 缺少 if n % 2 == 0: return False</code></pre>
|
||||
<p>official test 触发 assertion failure；流畅说明与闭合 code fence 都没有替它兜底。</p>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="four-ledgers">
|
||||
<article class="done"><span>01</span><b>STOPPING</b><p>EOS / budget cap</p></article>
|
||||
<i>→</i>
|
||||
<article class="done"><span>02</span><b>TASK TERMINAL</b><p>final marker / fence</p></article>
|
||||
<i>→</i>
|
||||
<article class="done"><span>03</span><b>COVERAGE</b><p>number / AST + sandbox</p></article>
|
||||
<i>→</i>
|
||||
<article><span>04</span><b>CORRECTNESS</b><p>gold / official tests</p></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="sp-panel" data-sp-panel="edges" hidden>
|
||||
<div class="sp-panel-lead">
|
||||
<div><span>III / SET-TO-SET COMPARISON</span><h4>不要拿一对随机回答，冒充两个条件的分布</h4></div>
|
||||
<p>
|
||||
aligned 看同 replicate label 的八对；nearest 让每条样本去另一侧寻找最接近轨迹，
|
||||
再双向平均。两者都只是八样本描述量。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="sp-controls edge-controls">
|
||||
<label><span>SOURCE</span>
|
||||
<select data-sp-edge-source aria-label="选择 edge source">
|
||||
{lab.sources.map((source: any) => (
|
||||
<option value={source.id}>{sourceLabels[source.id]}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label><span>EDGE</span>
|
||||
<select data-sp-edge aria-label="选择采样 edge">
|
||||
{edges.map((edge) => (
|
||||
<option value={edge}>{lab.contract.edgeLabels[edge]}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<div class="sampling-config"><span>PAIRING STATUS</span><b>BATCH-SEED ALIGNED ≠ CRN</b></div>
|
||||
</div>
|
||||
|
||||
<div class="set-metrics">
|
||||
<article><span>ALIGNED MEAN</span><b data-sp-edge-aligned>—</b><i><u data-sp-edge-aligned-bar></u></i></article>
|
||||
<article><span>SYMMETRIC NEAREST</span><b data-sp-edge-nearest>—</b><i><u data-sp-edge-nearest-bar></u></i></article>
|
||||
<article><span>EXACT HASH ∩ / ∪</span><b data-sp-edge-overlap>—</b><p>完整 token trajectory</p></article>
|
||||
</div>
|
||||
|
||||
<div class="two-sets">
|
||||
<article>
|
||||
<header><span>LEFT SAMPLE SET</span><b data-sp-edge-left-label>—</b></header>
|
||||
<div data-sp-edge-left-set></div>
|
||||
</article>
|
||||
<div class="set-bridge"><b>8 ↔ 8</b><span>双向找最近邻</span><i></i></div>
|
||||
<article>
|
||||
<header><span>RIGHT SAMPLE SET</span><b data-sp-edge-right-label>—</b></header>
|
||||
<div data-sp-edge-right-set></div>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="edge-overview">
|
||||
<header><b>EDGE · FOUR-SOURCE MEAN</b><b>ALIGNED</b><b>NEAREST</b><b>EXACT ∩ / ∪</b></header>
|
||||
{edges.map((edge) => {
|
||||
const row = lab.edges[edge];
|
||||
return (
|
||||
<div>
|
||||
<span>{row.label}</span>
|
||||
<b>{(row.meanAlignedSimilarity * 100).toFixed(1)}%</b>
|
||||
<b>{(row.meanSymmetricNearestSimilarity * 100).toFixed(1)}%</b>
|
||||
<b>{row.exactHashIntersections} / {row.exactHashUnion}</b>
|
||||
<i style={`--edge-nearest:${row.meanSymmetricNearestSimilarity * 100}%`}></i>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<aside class="sp-warning dark">
|
||||
<b>为什么 nearest 总比 aligned 高？</b>
|
||||
<p>
|
||||
nearest 主动从另一侧八条里挑最像的一条；aligned 没有这个选择自由。
|
||||
因此差值不是“条件效应”，nearest 也不是无偏分布距离。
|
||||
</p>
|
||||
</aside>
|
||||
</section>
|
||||
|
||||
<section class="sp-panel" data-sp-panel="reproduction" hidden>
|
||||
<div class="sp-panel-lead">
|
||||
<div><span>IV / RANDOMNESS × REPRODUCTION</span><h4>“会变化”和“可复现”必须同时成立</h4></div>
|
||||
<p>
|
||||
不同 seed 应该分叉；相同 checkpoint、输入、行顺序、软件与 seed 则应该回到同一条
|
||||
token trajectory。两者不是矛盾，而是采样实验的两道独立闸门。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="randomness-equation">
|
||||
<article><span>DIFFERENT RUN SEED</span><b>R0 ≠ R1</b><p>31 / 32 同格 trajectory 分叉</p></article>
|
||||
<i>+</i>
|
||||
<article><span>SAME FULL CONTRACT</span><b>R0 = R0′</b><p>新进程完整 token IDs exact</p></article>
|
||||
<i>=</i>
|
||||
<article class="result"><span>REPRODUCIBLE SAMPLING</span><b>64 / 64</b><p>随机轨迹不是环境漂移</p></article>
|
||||
</div>
|
||||
|
||||
<div class="seed-derivation">
|
||||
<header><span>SEED CONTRACT</span><b>结果之前冻结，不手选“好看 seed”</b></header>
|
||||
<div>
|
||||
<article><span>01 / BASE</span><b>SHA256(protocol / seed / index)</b><p>取前 4 bytes</p></article>
|
||||
<i>→</i>
|
||||
<article><span>02 / SOURCE</span><b>SHA256(protocol · base · source ID)</b><p>取前 8 bytes mod 2⁶³−1</p></article>
|
||||
<i>→</i>
|
||||
<article><span>03 / RNG</span><b>torch + CUDA manual seed</b><p>记录 pre/post state hash</p></article>
|
||||
<i>→</i>
|
||||
<article><span>04 / BATCH</span><b>固定八行顺序</b><p>顺序改变即新协议</p></article>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="repro-fields">
|
||||
{[
|
||||
["RUN SEED", "64 / 64"],
|
||||
["PROMPT HASH", "64 / 64"],
|
||||
["TOKEN IDS", "64 / 64"],
|
||||
["DECODED TEXT", "64 / 64"],
|
||||
["EOS STATE", "64 / 64"],
|
||||
["TRUNCATION", "64 / 64"],
|
||||
["CPU RNG PRE", "64 / 64"],
|
||||
["CUDA RNG PRE", "64 / 64"],
|
||||
].map(([label, value]) => (
|
||||
<article><span>{label}</span><b>{value}</b><i>EXACT</i></article>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div class="environment-lock">
|
||||
<article><span>CHECKPOINT</span><b>85864749…f64c7</b><p>12 个模型文件逐 SHA-256</p></article>
|
||||
<article><span>SOFTWARE</span><b>Torch 2.11 · TF 4.41.2</b><p>generation utils 也记录 hash</p></article>
|
||||
<article><span>PLACEMENT</span><b>CUDA L0–23 · CPU L24–26</b><p>final norm + LM head offload</p></article>
|
||||
<article><span>SCOPE</span><b>R0 / R1 only</b><p>不能扩大写成 256 / 256</p></article>
|
||||
</div>
|
||||
|
||||
<aside class="sp-warning">
|
||||
<b>exact replay 不是跨版本承诺</b>
|
||||
<p>
|
||||
更换 PyTorch、Transformers、CUDA kernel、硬件或 batch 行顺序都可能改变轨迹。
|
||||
这里证明的是固定执行合同内的可复现性。
|
||||
</p>
|
||||
</aside>
|
||||
</section>
|
||||
|
||||
<figcaption>
|
||||
<b>证据边界</b>
|
||||
<span>
|
||||
4 sources × 8 seeds 不是 benchmark，也没有恢复完整生成分布。BOS / x / 句点格不是
|
||||
官方有效聊天格式；unique hash 不是语义多样性；batch-seed aligned 不是逐行共享随机数。
|
||||
</span>
|
||||
<code>FORMAL 46c7edf…45af · EVAL 078f486…c4d8 · RERUN 72d050e…f6a0</code>
|
||||
</figcaption>
|
||||
|
||||
<script is:inline type="application/json" data-sp-data set:html={json}></script>
|
||||
</figure>
|
||||
|
||||
<script>
|
||||
document.querySelectorAll<HTMLElement>("[data-sampling-lab]").forEach((root) => {
|
||||
const payload = root.querySelector<HTMLScriptElement>("[data-sp-data]");
|
||||
if (!payload) return;
|
||||
const data = JSON.parse(payload.textContent ?? "{}");
|
||||
const one = <T extends Element>(selector: string) => (
|
||||
root.querySelector<T>(selector)
|
||||
);
|
||||
const set = (selector: string, value: string) => {
|
||||
const node = one<HTMLElement>(selector);
|
||||
if (node) node.textContent = value;
|
||||
};
|
||||
const pct = (value: number, digits = 1) => (
|
||||
`${(value * 100).toFixed(digits)}%`
|
||||
);
|
||||
const labels: Record<string, string> = {
|
||||
s0_eos: "S0 · EOS",
|
||||
s1_eos: "S1 · EOS",
|
||||
s0_bos: "S0 · BOS",
|
||||
s1_bos: "S1 · BOS",
|
||||
s0_x: "S0 · x",
|
||||
s1_x: "S1 · x",
|
||||
s0_period: "S0 · 句点",
|
||||
s1_period: "S1 · 句点",
|
||||
};
|
||||
const sourceById = new Map(
|
||||
data.sources.map((source: any) => [source.id, source]),
|
||||
);
|
||||
|
||||
const tabs = [
|
||||
...root.querySelectorAll<HTMLButtonElement>("[data-sp-tab]"),
|
||||
];
|
||||
const panels = [
|
||||
...root.querySelectorAll<HTMLElement>("[data-sp-panel]"),
|
||||
];
|
||||
tabs.forEach((button, index) => {
|
||||
button.addEventListener("click", () => {
|
||||
const target = button.dataset.spTab;
|
||||
tabs.forEach((candidate) => {
|
||||
const active = candidate === button;
|
||||
candidate.setAttribute("aria-selected", String(active));
|
||||
candidate.tabIndex = active ? 0 : -1;
|
||||
});
|
||||
panels.forEach((panel) => {
|
||||
panel.hidden = panel.dataset.spPanel !== target;
|
||||
});
|
||||
});
|
||||
button.addEventListener("keydown", (event) => {
|
||||
if (!["ArrowLeft", "ArrowRight"].includes(event.key)) return;
|
||||
event.preventDefault();
|
||||
const delta = event.key === "ArrowRight" ? 1 : -1;
|
||||
tabs[(index + delta + tabs.length) % tabs.length].click();
|
||||
tabs[(index + delta + tabs.length) % tabs.length].focus();
|
||||
});
|
||||
});
|
||||
|
||||
const sourceSelect = one<HTMLSelectElement>("[data-sp-source]");
|
||||
const conditionSelect = one<HTMLSelectElement>("[data-sp-condition]");
|
||||
const seedContainer = one<HTMLElement>("[data-sp-seeds]");
|
||||
const renderFocus = (sample: any, repeated: boolean) => {
|
||||
set(
|
||||
"[data-sp-focus-seed]",
|
||||
`${sample.replicate} · base ${sample.baseSeed.toLocaleString()}`,
|
||||
);
|
||||
set(
|
||||
"[data-sp-focus-meta]",
|
||||
`${sample.generatedTokens} TOKENS · ${sample.hitEos ? "NATURAL EOS" : "BUDGET TRUNCATED"}${repeated ? " · REPEATED HASH" : ""}`,
|
||||
);
|
||||
set(
|
||||
"[data-sp-focus-preview]",
|
||||
sample.preview || "（空文本）",
|
||||
);
|
||||
set(
|
||||
"[data-sp-focus-hash]",
|
||||
`trajectory sha256 · ${sample.trajectoryHash}`,
|
||||
);
|
||||
};
|
||||
const renderTrajectories = () => {
|
||||
const source = sourceById.get(sourceSelect?.value) as any;
|
||||
const condition = conditionSelect?.value ?? "s0_eos";
|
||||
if (!source) return;
|
||||
const row = source.conditions[condition];
|
||||
set("[data-sp-unique]", `${row.uniqueTrajectories} / 8`);
|
||||
set("[data-sp-eos]", `${row.naturalEos} / 8`);
|
||||
set("[data-sp-greedy]", row.greedyInSamples ? "YES" : "NO");
|
||||
set(
|
||||
"[data-sp-pairwise]",
|
||||
`${pct(row.pairwiseSimilarity.mean)} · ${pct(row.pairwiseSimilarity.min)}–${pct(row.pairwiseSimilarity.max)}`,
|
||||
);
|
||||
set(
|
||||
"[data-sp-stage-title]",
|
||||
`${source.id} · ${labels[condition]}`,
|
||||
);
|
||||
const counts = new Map<string, number>();
|
||||
row.trajectories.forEach((sample: any) => {
|
||||
counts.set(
|
||||
sample.trajectoryHash,
|
||||
(counts.get(sample.trajectoryHash) ?? 0) + 1,
|
||||
);
|
||||
});
|
||||
if (!seedContainer) return;
|
||||
const buttons = row.trajectories.map((sample: any, index: number) => {
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.className = sample.hitEos ? "eos" : "truncated";
|
||||
const repeated = (counts.get(sample.trajectoryHash) ?? 0) > 1;
|
||||
if (repeated) button.classList.add("repeat");
|
||||
button.setAttribute(
|
||||
"aria-label",
|
||||
`${sample.replicate}, ${sample.generatedTokens} tokens`,
|
||||
);
|
||||
const label = document.createElement("span");
|
||||
label.textContent = sample.replicate;
|
||||
const chart = document.createElement("i");
|
||||
chart.style.setProperty(
|
||||
"--height",
|
||||
`${Math.max(4, sample.generatedTokens / 512 * 100)}%`,
|
||||
);
|
||||
const meta = document.createElement("b");
|
||||
meta.textContent = String(sample.generatedTokens);
|
||||
const hash = document.createElement("code");
|
||||
hash.textContent = sample.trajectoryHash.slice(0, 7);
|
||||
button.append(label, chart, meta, hash);
|
||||
button.addEventListener("click", () => {
|
||||
buttons.forEach((candidate: HTMLButtonElement) => (
|
||||
candidate.classList.toggle("selected", candidate === button)
|
||||
));
|
||||
renderFocus(sample, repeated);
|
||||
});
|
||||
if (index === 0) button.classList.add("selected");
|
||||
return button;
|
||||
});
|
||||
seedContainer.replaceChildren(...buttons);
|
||||
renderFocus(
|
||||
row.trajectories[0],
|
||||
(counts.get(row.trajectories[0].trajectoryHash) ?? 0) > 1,
|
||||
);
|
||||
};
|
||||
sourceSelect?.addEventListener("change", renderTrajectories);
|
||||
conditionSelect?.addEventListener("change", renderTrajectories);
|
||||
|
||||
const taskCondition = one<HTMLSelectElement>("[data-sp-task-condition]");
|
||||
const renderTaskCells = (
|
||||
domain: "math" | "code",
|
||||
containerSelector: string,
|
||||
totalSelector: string,
|
||||
) => {
|
||||
const source = data.sources.find(
|
||||
(candidate: any) => candidate.domain === domain,
|
||||
);
|
||||
const condition = taskCondition?.value ?? "s0_eos";
|
||||
const rows = source.conditions[condition].trajectories;
|
||||
const container = one<HTMLElement>(containerSelector);
|
||||
if (!container) return;
|
||||
let passing = 0;
|
||||
const cells = rows.map((sample: any) => {
|
||||
const article = document.createElement("article");
|
||||
const passed = domain === "math"
|
||||
? sample.math?.exact
|
||||
: sample.code?.passed;
|
||||
passing += Number(Boolean(passed));
|
||||
article.className = passed ? "pass" : "fail";
|
||||
const seed = document.createElement("span");
|
||||
seed.textContent = sample.replicate;
|
||||
const result = document.createElement("b");
|
||||
result.textContent = domain === "math"
|
||||
? `${sample.math?.predicted ?? "—"}${passed ? " ✓" : " ✕"}`
|
||||
: `${(sample.code?.status ?? "not run").replaceAll("_", " ")}${passed ? " ✓" : " ✕"}`;
|
||||
const meta = document.createElement("em");
|
||||
meta.textContent = `${sample.generatedTokens} TOKENS · ${sample.hitEos ? "EOS" : "CAP"}`;
|
||||
article.append(seed, result, meta);
|
||||
return article;
|
||||
});
|
||||
container.replaceChildren(...cells);
|
||||
set(totalSelector, `${passing} / ${rows.length} PASS`);
|
||||
};
|
||||
const renderTasks = () => {
|
||||
renderTaskCells(
|
||||
"math",
|
||||
"[data-sp-math-cells]",
|
||||
"[data-sp-math-total]",
|
||||
);
|
||||
renderTaskCells(
|
||||
"code",
|
||||
"[data-sp-code-cells]",
|
||||
"[data-sp-code-total]",
|
||||
);
|
||||
};
|
||||
taskCondition?.addEventListener("change", renderTasks);
|
||||
|
||||
const edgeSource = one<HTMLSelectElement>("[data-sp-edge-source]");
|
||||
const edgeSelect = one<HTMLSelectElement>("[data-sp-edge]");
|
||||
const renderSet = (
|
||||
samples: any[],
|
||||
otherHashes: Set<string>,
|
||||
selector: string,
|
||||
) => {
|
||||
const container = one<HTMLElement>(selector);
|
||||
if (!container) return;
|
||||
const cells = samples.map((sample) => {
|
||||
const article = document.createElement("article");
|
||||
article.className = otherHashes.has(sample.trajectoryHash)
|
||||
? "overlap"
|
||||
: "";
|
||||
const seed = document.createElement("span");
|
||||
seed.textContent = sample.replicate;
|
||||
const hash = document.createElement("b");
|
||||
hash.textContent = sample.trajectoryHash.slice(0, 8);
|
||||
const meta = document.createElement("em");
|
||||
meta.textContent = `${sample.generatedTokens}T · ${sample.hitEos ? "EOS" : "CAP"}`;
|
||||
article.append(seed, hash, meta);
|
||||
return article;
|
||||
});
|
||||
container.replaceChildren(...cells);
|
||||
};
|
||||
const renderEdge = () => {
|
||||
const source = sourceById.get(edgeSource?.value) as any;
|
||||
const edgeName = edgeSelect?.value ?? "system_eos";
|
||||
if (!source) return;
|
||||
const row = source.edges[edgeName];
|
||||
const left = source.conditions[row.left];
|
||||
const right = source.conditions[row.right];
|
||||
set(
|
||||
"[data-sp-edge-aligned]",
|
||||
pct(row.batch_seed_aligned_similarity.mean),
|
||||
);
|
||||
set(
|
||||
"[data-sp-edge-nearest]",
|
||||
pct(row.symmetric_mean_nearest_neighbor_similarity),
|
||||
);
|
||||
set(
|
||||
"[data-sp-edge-overlap]",
|
||||
`${row.generated_hash_set_intersection} / ${row.generated_hash_set_union}`,
|
||||
);
|
||||
const alignedBar = one<HTMLElement>("[data-sp-edge-aligned-bar]");
|
||||
const nearestBar = one<HTMLElement>("[data-sp-edge-nearest-bar]");
|
||||
if (alignedBar) alignedBar.style.width = pct(
|
||||
row.batch_seed_aligned_similarity.mean,
|
||||
);
|
||||
if (nearestBar) nearestBar.style.width = pct(
|
||||
row.symmetric_mean_nearest_neighbor_similarity,
|
||||
);
|
||||
set("[data-sp-edge-left-label]", labels[row.left]);
|
||||
set("[data-sp-edge-right-label]", labels[row.right]);
|
||||
renderSet(
|
||||
left.trajectories,
|
||||
new Set(right.trajectories.map(
|
||||
(sample: any) => sample.trajectoryHash,
|
||||
)),
|
||||
"[data-sp-edge-left-set]",
|
||||
);
|
||||
renderSet(
|
||||
right.trajectories,
|
||||
new Set(left.trajectories.map(
|
||||
(sample: any) => sample.trajectoryHash,
|
||||
)),
|
||||
"[data-sp-edge-right-set]",
|
||||
);
|
||||
};
|
||||
edgeSource?.addEventListener("change", renderEdge);
|
||||
edgeSelect?.addEventListener("change", renderEdge);
|
||||
|
||||
renderTrajectories();
|
||||
renderTasks();
|
||||
renderEdge();
|
||||
});
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.sampling-lab {
|
||||
--ink: #182c33;
|
||||
--muted: #62706e;
|
||||
--line: rgba(25, 43, 50, .14);
|
||||
--paper: #f7f3e9;
|
||||
--teal: #2f7d70;
|
||||
--orange: #b35935;
|
||||
margin: 2.2rem 0 0;
|
||||
overflow: hidden;
|
||||
color: var(--ink);
|
||||
border: 1px solid var(--line);
|
||||
background: var(--paper);
|
||||
box-shadow: 0 28px 70px rgba(22, 35, 43, .1);
|
||||
}
|
||||
.sp-head {
|
||||
display: grid;
|
||||
grid-template-columns: 1.08fr .92fr;
|
||||
gap: 2.4rem;
|
||||
align-items: end;
|
||||
padding: 2rem;
|
||||
color: #fff;
|
||||
background:
|
||||
radial-gradient(circle at 78% 18%, rgba(212, 124, 73, .3), transparent 27%),
|
||||
radial-gradient(circle at 18% 90%, rgba(79, 165, 147, .23), transparent 30%),
|
||||
linear-gradient(137deg, #112a33, #28444a 64%, #3f4b49);
|
||||
}
|
||||
.sp-head p { margin: 0; color: rgba(255,255,255,.74); font-size: .76rem; line-height: 1.72; }
|
||||
.sp-head > div > p { color: #8ed1c5; font: 760 .6rem/1.2 var(--font-mono); letter-spacing: .11em; }
|
||||
.sp-head h3 { max-width: 680px; margin: .72rem 0 0; color: #fff; font: 760 clamp(1.55rem,3vw,2.45rem)/1.14 var(--font-display); }
|
||||
.sp-head code { padding: .08rem .25rem; color: #f3d3bd; background: rgba(0,0,0,.16); font: .64rem/1.2 var(--font-mono); }
|
||||
.sp-ledger { display: grid; grid-template-columns: repeat(6,1fr); border-bottom: 1px solid var(--line); }
|
||||
.sp-ledger article { min-width: 0; padding: 1rem; border-right: 1px solid var(--line); background: #ebe7dc; }
|
||||
.sp-ledger article:last-child { border-right: 0; }
|
||||
.sp-ledger article.pass { background: rgba(48,126,111,.12); }
|
||||
.sp-ledger span,
|
||||
.trajectory-summary span,
|
||||
.set-metrics span,
|
||||
.environment-lock span { display: block; color: #61706d; font: 730 .5rem/1.25 var(--font-mono); letter-spacing: .07em; }
|
||||
.sp-ledger b { display: block; margin-top: .45rem; font: 780 .82rem/1.2 var(--font-mono); }
|
||||
.sp-ledger p { margin: .34rem 0 0; color: #69736f; font-size: .56rem; line-height: 1.45; }
|
||||
.sp-tabs { display: grid; grid-template-columns: repeat(4,1fr); border-bottom: 1px solid var(--line); }
|
||||
.sp-tabs button { display: grid; grid-template-columns: auto 1fr; grid-template-rows: auto auto; column-gap: .72rem; min-width: 0; padding: .92rem 1rem; color: #263a40; border: 0; border-right: 1px solid var(--line); background: #fbf8f1; text-align: left; cursor: pointer; }
|
||||
.sp-tabs button:last-child { border-right: 0; }
|
||||
.sp-tabs button[aria-selected="true"] { color: #fff; background: var(--orange); }
|
||||
.sp-tabs span { grid-row: 1 / 3; opacity: .66; font: 720 .54rem/1.2 var(--font-mono); }
|
||||
.sp-tabs b { font: 730 .73rem/1.25 var(--font-display); }
|
||||
.sp-tabs small { opacity: .68; font: .51rem/1.3 var(--font-mono); }
|
||||
.sp-panel { padding: 1.55rem; }
|
||||
.sp-panel[hidden] { display: none; }
|
||||
.sp-panel-lead { display: grid; grid-template-columns: 1fr 1fr; gap: 2rem; align-items: end; margin-bottom: 1.2rem; }
|
||||
.sp-panel-lead span { color: var(--orange); font: 750 .55rem/1.2 var(--font-mono); letter-spacing: .09em; }
|
||||
.sp-panel-lead h4 { margin: .35rem 0 0; font: 750 1.24rem/1.2 var(--font-display); }
|
||||
.sp-panel-lead p { margin: 0; color: var(--muted); font-size: .68rem; line-height: 1.68; }
|
||||
.sp-controls { display: grid; grid-template-columns: 1fr 1fr .82fr; gap: 1px; border: 1px solid var(--line); background: var(--line); }
|
||||
.sp-controls > * { min-width: 0; padding: .76rem; background: #ece8de; }
|
||||
.sp-controls span,
|
||||
.task-condition-control span { display: block; margin-bottom: .42rem; color: #65716f; font: 720 .5rem/1.2 var(--font-mono); letter-spacing: .06em; }
|
||||
.sp-controls select,
|
||||
.task-condition-control select { width: 100%; min-width: 0; padding: .58rem; color: #24383e; border: 1px solid rgba(30,38,43,.2); background: #fffdf8; font: 670 .6rem/1.3 var(--font-mono); }
|
||||
.sampling-config { display: flex; flex-direction: column; justify-content: center; }
|
||||
.sampling-config b { color: #315d58; font: 720 .62rem/1.45 var(--font-mono); }
|
||||
.trajectory-summary { display: grid; grid-template-columns: repeat(4,1fr); gap: 1px; margin-top: 1rem; border: 1px solid var(--line); background: var(--line); }
|
||||
.trajectory-summary article { padding: .8rem; background: #f0ece2; }
|
||||
.trajectory-summary b { display: block; margin-top: .38rem; color: #2b6d62; font: 780 .75rem/1.2 var(--font-mono); }
|
||||
.trajectory-summary p { margin: .35rem 0 0; color: #6c7571; font-size: .54rem; }
|
||||
.seed-stage { margin-top: 1rem; border: 1px solid var(--line); background: #eee9df; }
|
||||
.seed-stage > header { display: flex; justify-content: space-between; gap: 1rem; align-items: center; padding: .75rem .85rem; color: #e8f0ed; background: #29434a; }
|
||||
.seed-stage header span { display: block; color: #91c9c0; font: 700 .47rem/1.2 var(--font-mono); }
|
||||
.seed-stage header b { display: block; margin-top: .25rem; font: 690 .62rem/1.2 var(--font-mono); }
|
||||
.seed-stage header p { margin: 0; font: .5rem/1.4 var(--font-mono); }
|
||||
.seed-stage header p i { display: inline-block; width: .55rem; height: .55rem; margin: 0 .25rem 0 .6rem; vertical-align: -.08rem; background: #3f9b89; }
|
||||
.seed-stage header p i:first-child { margin-left: 0; }
|
||||
.seed-stage header p i.truncated { background: #b45a37; }
|
||||
.seed-stage header p i.repeat { background: #d4a44d; }
|
||||
.seed-bars { display: grid; grid-template-columns: repeat(8,1fr); gap: 1px; height: 13rem; background: var(--line); }
|
||||
.seed-bars button { position: relative; display: grid; grid-template-rows: auto 1fr auto auto; gap: .35rem; min-width: 0; padding: .62rem .48rem; color: #31464b; border: 0; background: #f7f3ea; cursor: pointer; }
|
||||
.seed-bars button.selected { background: #fffdf8; box-shadow: inset 0 0 0 2px var(--orange); }
|
||||
.seed-bars button > span { font: 760 .56rem/1 var(--font-mono); }
|
||||
.seed-bars button > i { position: relative; align-self: end; width: 100%; height: var(--height); max-height: 8.2rem; background: linear-gradient(to top, #2f7d70, #76b6aa); }
|
||||
.seed-bars button.truncated > i { background: linear-gradient(to top, #a94f31, #d68d63); }
|
||||
.seed-bars button.repeat > i::after { position: absolute; inset: .25rem; border: 1px dashed #fff6d9; content: ""; }
|
||||
.seed-bars button > b { font: 740 .6rem/1 var(--font-mono); }
|
||||
.seed-bars button > code { overflow: hidden; color: #77807c; font: .43rem/1 var(--font-mono); text-overflow: ellipsis; }
|
||||
.seed-focus { display: grid; grid-template-columns: .34fr 1fr; gap: 1rem; padding: .9rem; border-top: 1px solid var(--line); background: #fbf8f1; }
|
||||
.seed-focus span { color: var(--orange); font: 750 .53rem/1.2 var(--font-mono); }
|
||||
.seed-focus b { display: block; margin-top: .4rem; color: #345b57; font: 690 .55rem/1.45 var(--font-mono); }
|
||||
.seed-focus p { margin: 0; color: #53615f; font-size: .62rem; line-height: 1.55; }
|
||||
.seed-focus code { grid-column: 1 / -1; overflow-wrap: anywhere; color: #737d79; font: .46rem/1.4 var(--font-mono); }
|
||||
.condition-overview,
|
||||
.edge-overview { margin-top: 1rem; border: 1px solid var(--line); }
|
||||
.condition-overview > header,
|
||||
.condition-overview > div { display: grid; grid-template-columns: 1.25fr repeat(4,1fr); gap: .65rem; align-items: center; padding: .62rem .8rem; border-bottom: 1px solid rgba(30,38,43,.1); }
|
||||
.condition-overview > header { color: #e5edeb; background: #29434a; font: 680 .46rem/1.2 var(--font-mono); }
|
||||
.condition-overview > div:last-child { border-bottom: 0; }
|
||||
.condition-overview > div:nth-child(odd) { background: #ede9df; }
|
||||
.condition-overview span { font: 680 .59rem/1.2 var(--font-mono); }
|
||||
.condition-overview b { color: #5d6865; font: 670 .57rem/1.2 var(--font-mono); }
|
||||
.condition-overview b.good { color: #277563; }
|
||||
.condition-overview b.warn { color: #a95131; }
|
||||
.sp-warning { display: grid; grid-template-columns: .35fr 1fr; gap: 1rem; margin-top: 1rem; padding: .85rem; color: #5c4a3b; border: 1px solid rgba(177,91,50,.2); background: rgba(183,98,53,.09); }
|
||||
.sp-warning b { font: 730 .63rem/1.4 var(--font-display); }
|
||||
.sp-warning p { margin: 0; font-size: .61rem; line-height: 1.6; }
|
||||
.sp-warning.dark { color: #dfe9e6; border: 0; background: #29434a; }
|
||||
.task-condition-control { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; border: 1px solid var(--line); background: var(--line); }
|
||||
.task-condition-control > * { min-width: 0; padding: .78rem; background: #ede9df; }
|
||||
.task-condition-control > div { display: flex; flex-direction: column; justify-content: center; }
|
||||
.task-condition-control > div b { color: #326a61; font: 740 .69rem/1.2 var(--font-mono); }
|
||||
.sample-task-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; margin-top: 1rem; border: 1px solid var(--line); background: var(--line); }
|
||||
.sample-task-grid > article { min-width: 0; padding: .95rem; background: #efebe1; }
|
||||
.sample-task-grid > article > header { display: flex; justify-content: space-between; align-items: end; gap: .8rem; }
|
||||
.sample-task-grid header span { display: block; color: #6a7571; font: 680 .49rem/1.2 var(--font-mono); }
|
||||
.sample-task-grid header b { display: block; margin-top: .28rem; color: #326d62; font: 750 .72rem/1.2 var(--font-mono); }
|
||||
.sample-task-grid header em { color: #8c5a43; font: 680 .52rem/1.2 var(--font-mono); }
|
||||
.seed-task-cells { display: grid; grid-template-columns: repeat(4,1fr); gap: 1px; margin-top: .72rem; background: var(--line); }
|
||||
.seed-task-cells article { min-width: 0; padding: .62rem; background: #fffaf2; box-shadow: inset 0 .22rem #ac5434; }
|
||||
.seed-task-cells article.pass { box-shadow: inset 0 .22rem #358274; }
|
||||
.seed-task-cells span,
|
||||
.seed-task-cells b,
|
||||
.seed-task-cells em { display: block; overflow-wrap: anywhere; }
|
||||
.seed-task-cells span { color: #727b77; font: .46rem/1.2 var(--font-mono); }
|
||||
.seed-task-cells b { margin-top: .32rem; font: 680 .55rem/1.25 var(--font-mono); }
|
||||
.seed-task-cells em { margin-top: .3rem; color: #77817d; font: .41rem/1.35 var(--font-mono); }
|
||||
.failure-cases { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; margin-top: 1rem; border: 1px solid rgba(174,82,47,.19); background: rgba(174,82,47,.19); }
|
||||
.failure-cases > article { min-width: 0; padding: 1rem; background: #f4e9df; }
|
||||
.failure-cases > article > span { color: #a04d31; font: 740 .49rem/1.2 var(--font-mono); letter-spacing: .08em; }
|
||||
.failure-cases h5 { margin: .45rem 0 .75rem; font: 730 .86rem/1.3 var(--font-display); }
|
||||
.failure-cases p { margin: .7rem 0 0; color: #665f58; font-size: .6rem; line-height: 1.55; }
|
||||
.failure-cases p code { font: .52rem/1.2 var(--font-mono); }
|
||||
.reasoning-bug { display: grid; grid-template-columns: 1fr auto 1fr; gap: .5rem; align-items: center; }
|
||||
.reasoning-bug b { padding: .62rem; color: #6a3d2e; background: rgba(174,82,47,.1); font: 680 .57rem/1.35 var(--font-mono); }
|
||||
.reasoning-bug i { color: #a74f31; font: 680 .45rem/1.2 var(--font-mono); }
|
||||
.failure-cases pre { overflow: auto; margin: 0; padding: .7rem; color: #dfe9e6; background: #293f45; font: .52rem/1.55 var(--font-mono); }
|
||||
.four-ledgers { display: grid; grid-template-columns: 1fr auto 1fr auto 1fr auto 1fr; gap: .45rem; align-items: center; margin-top: 1rem; }
|
||||
.four-ledgers article { padding: .75rem; border: 1px solid var(--line); background: #eeeae0; }
|
||||
.four-ledgers article.done { background: rgba(48,126,111,.11); }
|
||||
.four-ledgers article > span { color: var(--orange); font: 730 .47rem/1 var(--font-mono); }
|
||||
.four-ledgers article > b { display: block; margin-top: .35rem; font: 720 .58rem/1.2 var(--font-mono); }
|
||||
.four-ledgers article > p { margin: .3rem 0 0; color: #6a7470; font-size: .52rem; }
|
||||
.four-ledgers > i { color: var(--orange); font: 750 .7rem/1 var(--font-mono); }
|
||||
.set-metrics { display: grid; grid-template-columns: repeat(3,1fr); gap: 1px; margin-top: 1rem; border: 1px solid var(--line); background: var(--line); }
|
||||
.set-metrics article { padding: .85rem; background: #eeeae0; }
|
||||
.set-metrics b { display: block; margin-top: .38rem; color: #2e6d62; font: 760 .76rem/1.2 var(--font-mono); }
|
||||
.set-metrics article > i { display: block; height: .38rem; margin-top: .6rem; background: rgba(47,125,112,.16); }
|
||||
.set-metrics article > i > u { display: block; width: 0; height: 100%; background: var(--teal); text-decoration: none; transition: width .25s ease; }
|
||||
.set-metrics p { margin: .45rem 0 0; color: #6c7672; font-size: .53rem; }
|
||||
.two-sets { display: grid; grid-template-columns: 1fr 5.5rem 1fr; gap: 1px; margin-top: 1rem; border: 1px solid var(--line); background: var(--line); }
|
||||
.two-sets > article { min-width: 0; padding: .85rem; background: #efebe1; }
|
||||
.two-sets > article > header { display: flex; justify-content: space-between; gap: .7rem; }
|
||||
.two-sets header span { color: #717b77; font: .47rem/1.2 var(--font-mono); }
|
||||
.two-sets header b { color: #3a645e; font: 690 .54rem/1.2 var(--font-mono); }
|
||||
.two-sets article > div { display: grid; grid-template-columns: repeat(4,1fr); gap: 1px; margin-top: .68rem; background: var(--line); }
|
||||
.two-sets article > div article { min-width: 0; padding: .55rem; background: #fffaf2; }
|
||||
.two-sets article > div article.overlap { background: #d9eee7; box-shadow: inset 0 0 0 1px var(--teal); }
|
||||
.two-sets article > div span,
|
||||
.two-sets article > div b,
|
||||
.two-sets article > div em { display: block; overflow: hidden; text-overflow: ellipsis; }
|
||||
.two-sets article > div span { color: #77817d; font: .43rem/1.2 var(--font-mono); }
|
||||
.two-sets article > div b { margin-top: .3rem; font: 680 .49rem/1.2 var(--font-mono); }
|
||||
.two-sets article > div em { margin-top: .25rem; color: #7b817e; font: .39rem/1.2 var(--font-mono); }
|
||||
.set-bridge { display: grid; place-content: center; gap: .3rem; color: #e6eeeb; background: #29434a; text-align: center; }
|
||||
.set-bridge b { font: 760 .72rem/1 var(--font-mono); }
|
||||
.set-bridge span { font: .42rem/1.3 var(--font-mono); }
|
||||
.set-bridge i { display: block; height: 1px; background: #8dc8bd; }
|
||||
.edge-overview > header,
|
||||
.edge-overview > div { position: relative; display: grid; grid-template-columns: 1.6fr repeat(3,1fr); gap: .65rem; align-items: center; padding: .62rem .8rem; overflow: hidden; border-bottom: 1px solid rgba(30,38,43,.1); }
|
||||
.edge-overview > header { color: #e5edeb; background: #29434a; font: 680 .46rem/1.2 var(--font-mono); }
|
||||
.edge-overview > div:last-child { border-bottom: 0; }
|
||||
.edge-overview > div:nth-child(odd) { background: #ede9df; }
|
||||
.edge-overview span { z-index: 1; font: 650 .56rem/1.2 var(--font-mono); }
|
||||
.edge-overview b { z-index: 1; color: #5d6965; font: 670 .55rem/1.2 var(--font-mono); }
|
||||
.edge-overview > div > i { position: absolute; right: 0; bottom: 0; width: var(--edge-nearest); height: .16rem; background: var(--teal); opacity: .7; }
|
||||
.randomness-equation { display: grid; grid-template-columns: 1fr auto 1fr auto 1fr; gap: .65rem; align-items: center; }
|
||||
.randomness-equation article { padding: 1rem; border: 1px solid var(--line); background: #eeeae0; }
|
||||
.randomness-equation article.result { color: #e9f1ef; border: 0; background: #2d776c; }
|
||||
.randomness-equation span { display: block; opacity: .72; font: .48rem/1.2 var(--font-mono); }
|
||||
.randomness-equation b { display: block; margin-top: .45rem; font: 780 .95rem/1.2 var(--font-mono); }
|
||||
.randomness-equation p { margin: .4rem 0 0; opacity: .75; font-size: .55rem; }
|
||||
.randomness-equation > i { color: var(--orange); font: 780 1rem/1 var(--font-display); }
|
||||
.seed-derivation { margin-top: 1rem; border: 1px solid var(--line); }
|
||||
.seed-derivation > header { display: flex; justify-content: space-between; gap: 1rem; padding: .72rem .85rem; color: #e6eeeb; background: #29434a; }
|
||||
.seed-derivation > header span { color: #8fc9bd; font: .47rem/1.2 var(--font-mono); }
|
||||
.seed-derivation > header b { font: 670 .56rem/1.2 var(--font-mono); }
|
||||
.seed-derivation > div { display: grid; grid-template-columns: 1fr auto 1fr auto 1fr auto 1fr; gap: .45rem; align-items: center; padding: .9rem; background: #eeeae0; }
|
||||
.seed-derivation article { min-width: 0; padding: .65rem; border: 1px solid var(--line); background: #faf7ef; }
|
||||
.seed-derivation article span { color: var(--orange); font: .44rem/1.2 var(--font-mono); }
|
||||
.seed-derivation article b { display: block; margin-top: .35rem; overflow-wrap: anywhere; font: 650 .49rem/1.35 var(--font-mono); }
|
||||
.seed-derivation article p { margin: .32rem 0 0; color: #727b77; font-size: .47rem; }
|
||||
.seed-derivation > div > i { color: var(--orange); font: 720 .65rem/1 var(--font-mono); }
|
||||
.repro-fields { display: grid; grid-template-columns: repeat(4,1fr); gap: 1px; margin-top: 1rem; border: 1px solid var(--line); background: var(--line); }
|
||||
.repro-fields article { padding: .72rem; background: rgba(48,126,111,.1); }
|
||||
.repro-fields span { color: #5a716c; font: .46rem/1.2 var(--font-mono); }
|
||||
.repro-fields b { display: block; margin-top: .35rem; color: #296a5f; font: 740 .62rem/1.2 var(--font-mono); }
|
||||
.repro-fields i { display: block; margin-top: .28rem; color: #528078; font: .4rem/1.2 var(--font-mono); }
|
||||
.environment-lock { display: grid; grid-template-columns: repeat(4,1fr); gap: 1px; margin-top: 1rem; border: 1px solid var(--line); background: var(--line); }
|
||||
.environment-lock article { min-width: 0; padding: .78rem; background: #eeeae0; }
|
||||
.environment-lock b { display: block; margin-top: .38rem; overflow-wrap: anywhere; font: 690 .57rem/1.3 var(--font-mono); }
|
||||
.environment-lock p { margin: .35rem 0 0; color: #6c7571; font-size: .51rem; line-height: 1.45; }
|
||||
.sampling-lab > figcaption { display: grid; grid-template-columns: auto 1fr auto; gap: 1rem; align-items: center; padding: .9rem 1.1rem; color: #dfe8e6; background: #203a42; }
|
||||
.sampling-lab > figcaption b { color: #8dc8bd; font: 730 .53rem/1.2 var(--font-mono); }
|
||||
.sampling-lab > figcaption span { font-size: .54rem; line-height: 1.5; }
|
||||
.sampling-lab > figcaption code { color: #d9af95; font: .42rem/1.4 var(--font-mono); }
|
||||
@media (max-width: 900px) {
|
||||
.sp-head,
|
||||
.sp-panel-lead { grid-template-columns: 1fr; }
|
||||
.sp-ledger { grid-template-columns: repeat(3,1fr); }
|
||||
.sp-ledger article:nth-child(3) { border-right: 0; }
|
||||
.sp-tabs { grid-template-columns: 1fr 1fr; }
|
||||
.sp-controls,
|
||||
.sample-task-grid,
|
||||
.failure-cases { grid-template-columns: 1fr; }
|
||||
.two-sets { grid-template-columns: 1fr; }
|
||||
.set-bridge { min-height: 4rem; }
|
||||
.sampling-lab > figcaption { grid-template-columns: 1fr; }
|
||||
}
|
||||
@media (max-width: 640px) {
|
||||
.sp-panel { padding: 1rem; }
|
||||
.sp-head { padding: 1.35rem; }
|
||||
.sp-ledger,
|
||||
.trajectory-summary,
|
||||
.set-metrics,
|
||||
.repro-fields,
|
||||
.environment-lock { grid-template-columns: 1fr 1fr; }
|
||||
.sp-tabs { grid-template-columns: 1fr; }
|
||||
.seed-bars { grid-template-columns: repeat(4,1fr); height: 22rem; }
|
||||
.seed-focus { grid-template-columns: 1fr; }
|
||||
.condition-overview > header,
|
||||
.condition-overview > div {
|
||||
grid-template-columns: 1.2fr repeat(4,.72fr);
|
||||
gap: .25rem;
|
||||
min-width: 0;
|
||||
padding: .5rem .35rem;
|
||||
}
|
||||
.condition-overview > header b,
|
||||
.condition-overview > div b,
|
||||
.condition-overview > div span { font-size: .42rem; }
|
||||
.seed-task-cells { grid-template-columns: 1fr 1fr; }
|
||||
.four-ledgers,
|
||||
.randomness-equation,
|
||||
.seed-derivation > div { grid-template-columns: 1fr; }
|
||||
.four-ledgers > i,
|
||||
.randomness-equation > i,
|
||||
.seed-derivation > div > i { transform: rotate(90deg); text-align: center; }
|
||||
.edge-overview > header,
|
||||
.edge-overview > div {
|
||||
grid-template-columns: 1.45fr repeat(3,.65fr);
|
||||
gap: .25rem;
|
||||
min-width: 0;
|
||||
padding: .5rem .35rem;
|
||||
}
|
||||
.edge-overview > header b,
|
||||
.edge-overview > div b,
|
||||
.edge-overview > div span { font-size: .4rem; }
|
||||
}
|
||||
</style>
|
||||
|
||||
<style is:global>
|
||||
[data-sampling-lab] [data-sp-seeds] button {
|
||||
position: relative;
|
||||
display: grid;
|
||||
grid-template-rows: auto minmax(4rem, 1fr) auto auto;
|
||||
gap: .35rem;
|
||||
min-width: 0;
|
||||
padding: .62rem .48rem;
|
||||
overflow: hidden;
|
||||
color: #31464b;
|
||||
border: 0;
|
||||
background: #f7f3ea;
|
||||
cursor: pointer;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button.selected {
|
||||
background: #fffdf8;
|
||||
box-shadow: inset 0 0 0 2px #b35935;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button > span {
|
||||
font: 760 .56rem/1 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button > i {
|
||||
position: relative;
|
||||
display: block;
|
||||
align-self: end;
|
||||
width: 100%;
|
||||
height: var(--height);
|
||||
max-height: 8.2rem;
|
||||
background: linear-gradient(to top, #2f7d70, #76b6aa);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button.truncated > i {
|
||||
background: linear-gradient(to top, #a94f31, #d68d63);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button.repeat > i::after {
|
||||
position: absolute;
|
||||
inset: .25rem;
|
||||
border: 1px dashed #fff6d9;
|
||||
content: "";
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button > b {
|
||||
display: block;
|
||||
font: 740 .6rem/1 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-seeds] button > code {
|
||||
display: block;
|
||||
overflow: hidden;
|
||||
color: #77807c;
|
||||
font: .43rem/1 var(--font-mono);
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-math-cells] article,
|
||||
[data-sampling-lab] [data-sp-code-cells] article {
|
||||
min-width: 0;
|
||||
padding: .62rem;
|
||||
background: #fffaf2;
|
||||
box-shadow: inset 0 .22rem #ac5434;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-math-cells] article.pass,
|
||||
[data-sampling-lab] [data-sp-code-cells] article.pass {
|
||||
box-shadow: inset 0 .22rem #358274;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-math-cells] article > span,
|
||||
[data-sampling-lab] [data-sp-math-cells] article > b,
|
||||
[data-sampling-lab] [data-sp-math-cells] article > em,
|
||||
[data-sampling-lab] [data-sp-code-cells] article > span,
|
||||
[data-sampling-lab] [data-sp-code-cells] article > b,
|
||||
[data-sampling-lab] [data-sp-code-cells] article > em {
|
||||
display: block;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-math-cells] article > span,
|
||||
[data-sampling-lab] [data-sp-code-cells] article > span {
|
||||
color: #727b77;
|
||||
font: .46rem/1.2 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-math-cells] article > b,
|
||||
[data-sampling-lab] [data-sp-code-cells] article > b {
|
||||
margin-top: .32rem;
|
||||
font: 680 .55rem/1.25 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-math-cells] article > em,
|
||||
[data-sampling-lab] [data-sp-code-cells] article > em {
|
||||
margin-top: .3rem;
|
||||
color: #77817d;
|
||||
font: .41rem/1.35 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article {
|
||||
min-width: 0;
|
||||
padding: .55rem;
|
||||
background: #fffaf2;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article.overlap,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article.overlap {
|
||||
background: #d9eee7;
|
||||
box-shadow: inset 0 0 0 1px #2f7d70;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article > span,
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article > b,
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article > em,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article > span,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article > b,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article > em {
|
||||
display: block;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article > span,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article > span {
|
||||
color: #77817d;
|
||||
font: .43rem/1.2 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article > b,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article > b {
|
||||
margin-top: .3rem;
|
||||
font: 680 .49rem/1.2 var(--font-mono);
|
||||
}
|
||||
[data-sampling-lab] [data-sp-edge-left-set] article > em,
|
||||
[data-sampling-lab] [data-sp-edge-right-set] article > em {
|
||||
margin-top: .25rem;
|
||||
color: #7b817e;
|
||||
font: .39rem/1.2 var(--font-mono);
|
||||
}
|
||||
</style>
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,946 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"protocol_id": "llm-atlas-deepseek-chat-sampling-v1",
|
||||
"formal": {
|
||||
"path": "/tmp/deepseek-v2-lite-chat-sampling-formal.json",
|
||||
"sha256": "46c7edfce1409e798d6b0f06e905dd3a9d4ab6911acd8e42e2ec9d06d65345af",
|
||||
"content_hash": "7ed89b59e852136892e56da434b212e80a1f26f771e63bd6f92eeb8617a18ad5",
|
||||
"base_seeds": [
|
||||
19683830,
|
||||
1560062173,
|
||||
3978401375,
|
||||
1280933274,
|
||||
1467459869,
|
||||
1297359489,
|
||||
2722953988,
|
||||
3330978061
|
||||
]
|
||||
},
|
||||
"rerun": {
|
||||
"path": "/tmp/deepseek-v2-lite-chat-sampling-repro-r0r1.json",
|
||||
"sha256": "72d050eace531bb38fb2592e39dc7f1b9c9003306c92c568dcd4acaaa691f6a0",
|
||||
"content_hash": "f8a5fb017ab0e517b1df2899e388c47d07b4bd910912639df542703d1d9509e0",
|
||||
"base_seeds": [
|
||||
19683830,
|
||||
1560062173
|
||||
]
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "HumanEval/31",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "gsm8k/test/1069",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "tnews/test/4855",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 19683830,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s0_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_bos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_eos",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_period",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
},
|
||||
{
|
||||
"source_id": "wikitext2/raw-validation/0443",
|
||||
"base_seed": 1560062173,
|
||||
"condition": "s1_x",
|
||||
"run_seed_exact": true,
|
||||
"prompt_hash_exact": true,
|
||||
"generated_token_ids_exact": true,
|
||||
"decoded_text_exact": true,
|
||||
"eos_state_exact": true,
|
||||
"truncation_state_exact": true,
|
||||
"cpu_rng_pre_state_exact": true,
|
||||
"cuda_rng_pre_state_exact": true,
|
||||
"all_preregistered_fields_exact": true
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"cells": 64,
|
||||
"all_preregistered_fields_exact": 64,
|
||||
"by_field": {
|
||||
"run_seed_exact": 64,
|
||||
"prompt_hash_exact": 64,
|
||||
"generated_token_ids_exact": 64,
|
||||
"decoded_text_exact": 64,
|
||||
"eos_state_exact": 64,
|
||||
"truncation_state_exact": 64,
|
||||
"cpu_rng_pre_state_exact": 64,
|
||||
"cuda_rng_pre_state_exact": 64
|
||||
}
|
||||
},
|
||||
"claim_boundary": [
|
||||
"Only the rerun seed subset is independently reproduced.",
|
||||
"Exact replay is scoped to the pinned software and hardware contract.",
|
||||
"Reproduction does not imply trajectories are seed-invariant."
|
||||
],
|
||||
"content_hash": "d02d0a23efdea333fb0b04fc9a889c7f81eb755f8ef4a9c66f14226a230d94a3"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,7 @@ import DeepSeekLab from "@/components/DeepSeekLab.astro";
|
||||
import DeepSeekArtifactLab from "@/components/DeepSeekArtifactLab.astro";
|
||||
import DeepSeekBehaviorLab from "@/components/DeepSeekBehaviorLab.astro";
|
||||
import DeepSeekCompletionDepthLab from "@/components/DeepSeekCompletionDepthLab.astro";
|
||||
import DeepSeekSamplingLab from "@/components/DeepSeekSamplingLab.astro";
|
||||
import { deepseekBranches, deepseekLedgers, deepseekPaperChain, deepseekWaves } from "@/data/deepseek";
|
||||
|
||||
const toc = [
|
||||
@@ -33,21 +34,22 @@ const toc = [
|
||||
["22", "artifact", "真实权重执行"],
|
||||
["23", "behavior", "Chat：最终生成行为"],
|
||||
["24", "completion-depth", "Chat：完成度与全深度"],
|
||||
["25", "branches", "别漏掉旁支"],
|
||||
["26", "audit", "事实、推导与教学模型"],
|
||||
["25", "sampling", "Chat：多种子采样稳健性"],
|
||||
["26", "branches", "别漏掉旁支"],
|
||||
["27", "audit", "事实、推导与教学模型"],
|
||||
["↳", "papers", "六十节点阅读链"],
|
||||
];
|
||||
---
|
||||
|
||||
<BaseLayout
|
||||
title="DeepSeek 技术谱系与真实权重深读：从 Dense、MoE、MLA 到 R1 与 V4"
|
||||
description="用二十四张问题账、十次技术转向、十九个交互实验、真实 V2-Lite Base / Chat 权重、512-token 完成度评测、29 阶段隐藏状态与 26 层 MoE 路由追踪、吸收式缓存 trace 和六十个一手节点，完整理解 DeepSeek 的 MoE、MLA、FP8、DualPipe、GRPO、R1、V3.2 与 V4。"
|
||||
description="用二十四张问题账、十次技术转向、二十个交互实验、真实 V2-Lite Base / Chat 权重、512-token 完成度评测、29 阶段隐藏状态、26 层 MoE 路由追踪与 256 条多种子采样，完整理解 DeepSeek 的 MoE、MLA、FP8、DualPipe、GRPO、R1、V3.2 与 V4。"
|
||||
section="deepseek"
|
||||
>
|
||||
<header class="page-hero deepseek-hero">
|
||||
<div class="page-hero-inner">
|
||||
<div>
|
||||
<p class="eyebrow"><span>SPOTLIGHT / DEEPSEEK · ROUND 05</span> COMPLETION × FULL DEPTH × REAL WEIGHTS</p>
|
||||
<p class="eyebrow"><span>SPOTLIGHT / DEEPSEEK · ROUND 06</span> SAMPLING × COMPLETION × FULL DEPTH</p>
|
||||
<h1>不要背模型名<br />要看懂每次为什么转向</h1>
|
||||
<p class="lead">
|
||||
这不是七篇报告的摘要，而是一套可追问、可计算、可反驳的技术谱系：
|
||||
@@ -59,9 +61,9 @@ const toc = [
|
||||
<div><dt>SPAN</dt><dd>2024.01 → 2026.06</dd></div>
|
||||
<div><dt>LEDGERS</dt><dd>24 张问题账</dd></div>
|
||||
<div><dt>LINEAGE</dt><dd>10 次技术转向</dd></div>
|
||||
<div><dt>LABS</dt><dd>19 个可操作实验</dd></div>
|
||||
<div><dt>LABS</dt><dd>20 个可操作实验</dd></div>
|
||||
<div><dt>EVIDENCE</dt><dd>60 个一手 / 官方节点</dd></div>
|
||||
<div><dt>STATUS</dt><dd>五轮 · 全 27 层执行</dd></div>
|
||||
<div><dt>STATUS</dt><dd>六轮 · 256 条采样</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</header>
|
||||
@@ -810,8 +812,21 @@ const toc = [
|
||||
<DeepSeekCompletionDepthLab />
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="sampling">
|
||||
<p class="eyebrow"><span>25</span> GREEDY IS NOT A DISTRIBUTION</p>
|
||||
<h2>把单条最大概率路径打开：八个预注册 seed 下，轨迹集合是否仍然分叉</h2>
|
||||
<p class="lede">
|
||||
上一轮的 512-token 结果仍是 deterministic greedy：它只能看到每一步概率最大的
|
||||
一条路。这一轮冻结四条 source、八格 prompt batch 与八个 SHA-256 派生 seed，
|
||||
启用 checkpoint 随附的 <code>temperature=.3 / top_p=.95</code>，生成 256 条
|
||||
sampled outputs。停止、任务正确、集合相似度与新进程复现继续分账；
|
||||
batch-seed aligned 也明确不冒充逐行共享随机数的 paired causal design。
|
||||
</p>
|
||||
<DeepSeekSamplingLab />
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="branches">
|
||||
<p class="eyebrow"><span>25</span> THE MAIN LINE IS NOT THE WHOLE TREE</p>
|
||||
<p class="eyebrow"><span>26</span> THE MAIN LINE IS NOT THE WHOLE TREE</p>
|
||||
<h2>如果只读 V2 → V3 → R1 → V4，会漏掉五条反过来影响主线的旁支</h2>
|
||||
<div class="branch-grid">
|
||||
{deepseekBranches.map(([name, line, text, url]) => (
|
||||
@@ -831,7 +846,7 @@ const toc = [
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="audit">
|
||||
<p class="eyebrow"><span>26</span> EVIDENCE AUDIT</p>
|
||||
<p class="eyebrow"><span>27</span> EVIDENCE AUDIT</p>
|
||||
<h2>同一张页面里有三种知识，它们的语气必须不同</h2>
|
||||
<div class="audit-grid">
|
||||
<article class="reported">
|
||||
|
||||
@@ -145,18 +145,18 @@ const paths = [
|
||||
</a>
|
||||
<a class="release-card deepseek-release" href="/deepseek/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / DEEPSEEK ROUND 05</span> COMPLETION · TASK TESTS · FULL DEPTH</p>
|
||||
<p class="eyebrow"><span>NEW / DEEPSEEK ROUND 06</span> MULTI-SEED SAMPLING · EXACT REPLAY</p>
|
||||
<h2>从 Dense 到百万上下文：每次创新都在偿还上一代最贵的一张账</h2>
|
||||
<p>
|
||||
用二十四张问题账和十次技术转向走完 Dense→V4，再把官方 V2-Lite-Chat 的
|
||||
128 个输出统一延长到 512-token：121 个自然 EOS、Math / Code 真实 evaluator；
|
||||
同时沿 29 个隐藏阶段与 26 个 MoE gate 追踪 1,918,176 次目标路由决定。
|
||||
在 512-token greedy 与 29-stage / 26-gate 全深度 trace 之后，再按预注册的
|
||||
8 个 seed 生成 256 条官方 nucleus samples：251 条自然 EOS、242 条不同完整
|
||||
token 轨迹；Math 62 / 64、Code 63 / 64，并在新进程复现 R0/R1 的 64 / 64 格。
|
||||
</p>
|
||||
</div>
|
||||
<dl>
|
||||
<div><dt>LINEAGE</dt><dd>1991 → 2026 · 10 次转向</dd></div>
|
||||
<div><dt>NODES</dt><dd>60 个一手 / 官方节点</dd></div>
|
||||
<div><dt>LAB</dt><dd>19 · Base / Chat / full depth</dd></div>
|
||||
<div><dt>LAB</dt><dd>20 · Base / Chat / sampling</dd></div>
|
||||
</dl>
|
||||
<span class="release-arrow" aria-hidden="true">进入 DeepSeek 完整技术谱系 →</span>
|
||||
</a>
|
||||
|
||||
@@ -15,7 +15,7 @@ const workstreams = [
|
||||
{ label: "表示、位置与残差高速公路", value: 81, next: "加入真实 hidden-state / norm traces、长上下文位置外推复现与更多深层稳定性消融" },
|
||||
{ label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" },
|
||||
{ label: "数据工程与预训练配方", value: 73, next: "逐图精读 FineWeb / DCLM，加入真实去重与 mixture traces" },
|
||||
{ label: "DeepSeek 专题", value: 99, next: "扩大 completion/task 样本、sampling robustness 与干预式 mediation，再推进 SM90 FlashMLA、FP8/pipeline 与 R1-like RL" },
|
||||
{ label: "DeepSeek 专题", value: 99, next: "扩大 sampling 的 source/task 覆盖并推进干预式 mediation、SM90 FlashMLA、FP8/pipeline 与 R1-like RL" },
|
||||
{ label: "指令微调与人类偏好", value: 75, next: "加入真实偏好分歧样本、RM 长度偏置与 PPO/DPO 小模型复现" },
|
||||
{ label: "推理与测试时扩展", value: 76, next: "真实模型采样曲线、PRM 案例与逐篇图表精读" },
|
||||
{ label: "工具使用与长程 Agent", value: 74, next: "补真实环境 traces、cross-harness 对照、Agent RL 训练曲线与安全案例" },
|
||||
@@ -50,7 +50,7 @@ const workstreams = [
|
||||
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
|
||||
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
|
||||
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 19:55 CST</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-30 01:50 CST</dd></div>
|
||||
<div><dt>MODE</dt><dd>持续迭代，不锁死版本</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
@@ -97,12 +97,12 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
|
||||
<article><span>✓</span><h3>17 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
|
||||
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
|
||||
<article><span>✓</span><h3>八十六个原创交互视图</h3><p>K3 三轴图、八联报告实验与四联开放工件实验，DeepSeek 四联公式实验、十三联 Base 工件实验、Chat 行为与 completion/full-depth 两轮实验，以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>八十七个原创交互视图</h3><p>K3 三轴图、八联报告实验与四联开放工件实验，DeepSeek 四联公式实验、十三联 Base 工件实验、Chat 行为、completion/full-depth 与 multi-seed sampling 三轮实验，以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>十七篇首版长文</h3><p>K3、语言模型前史、Transformer、表示/位置/残差、DeepSeek、Scaling、数据工程、长上下文、MoE、后训练、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>语言模型前史深度专题</h3><p>八张独立问题账、33 个正式节点、20 段长文与概率—向量—记忆—对齐四联实验。</p></article>
|
||||
<article><span>✓</span><h3>Transformer 深度专题</h3><p>十张独立问题账、40 个正式节点、21 段正文与 QKV—Mask—多头位置—Block 成本四联实验。</p></article>
|
||||
<article><span>✓</span><h3>表示、位置与残差高速公路深度专题</h3><p>二十张问题账、66 个一手节点、DeepSeek/Kimi 双谱系，以及 Token—位置—Norm—Residual/FFN 四联实验。</p></article>
|
||||
<article><span>✓</span><h3>DeepSeek 五轮真实权重里程碑</h3><p>统一 512-token 预算让自然 EOS 从 31 / 128 增至 121 / 128；GSM8K strict exact 23 / 32、HumanEval 官方 tests pass 24 / 32，并保留四任务/域的样本边界。相同 Chat checkpoint 再执行 29-stage hidden 与 26-gate route trace；1,856 个 hidden hashes、1,664 个 route hashes 和 1,664 个 weight hashes 在新进程子集复跑中全部 exact。</p></article>
|
||||
<article><span>✓</span><h3>DeepSeek 六轮真实权重里程碑</h3><p>在 512-token greedy 与 29-stage / 26-gate 全深度 trace 之后，预注册 8 个 SHA-256 seed，生成 256 条 official nucleus samples：251 条 natural EOS、242 个 unique trajectory hashes；GSM8K 62 / 64 strict exact、HumanEval 63 / 64 tests pass，新进程 R0/R1 的八项合同字段 64 / 64 exact。</p></article>
|
||||
<article><span>✓</span><h3>Kimi K3 技术报告二轮深读</h3><p>三十二张问题账、Figure 1–16 / Table 1–5 审计、100 节点阅读链，以及 Delta—Decay—AttnRes—LatentMoE—SiTU—QB—MOPD—Cache 八联实验。</p></article>
|
||||
<article><span>✓</span><h3>Kimi K3 三轮开放工件里程碑</h3><p>固定官方 revisions，审计 96 个 shards、497,220 个 tensor entries 与真实 KDA / MLA / MoE / MoonViT shapes；四联实验分开显示层型、tensor anatomy、参数范围和复现边界。</p></article>
|
||||
<article><span>✓</span><h3>FlashKDA RTX 5090 执行闸门</h3><p>隔离 CUDA 13.0 / glibc 2.39 编译 sm_120a wheel；6/6 官方参考逐元素相等，并完成 fixed / varlen、三种 state mode 的 1,800 个 CUDA Event samples。</p></article>
|
||||
@@ -134,7 +134,7 @@ const workstreams = [
|
||||
<div class="queue-table">
|
||||
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
|
||||
<div><span>P0</span><strong>K3 三轮</strong><p>开放权重 traces → FlashKDA / AttnRes / MoE 真实行为 → Figure 1–16 数值重绘与独立复现</p><em>运行证据 + 逐图复现</em></div>
|
||||
<div><span>P0</span><strong>DeepSeek 五轮后续</strong><p>扩大任务与语言 source → sampling robustness → 干预式 mediation → SM90 FlashMLA / FP8 / pipeline traces → R1-like RL 小模型复现</p><em>运行证据 + 独立复现</em></div>
|
||||
<div><span>P0</span><strong>DeepSeek 六轮后续</strong><p>扩大任务与语言 source 的 sampling 覆盖 → 干预式 mediation → SM90 FlashMLA / FP8 / pipeline traces → R1-like RL 小模型复现</p><em>运行证据 + 独立复现</em></div>
|
||||
<div><span>P0</span><strong>Transformer 二轮</strong><p>多头电路逐图 → Pre/Post-LN 真实 traces → Flash/KV 配置与 kernel 对照</p><em>逐图笔记 + 实测边界</em></div>
|
||||
<div><span>P0</span><strong>表示、位置与残差二轮</strong><p>真实 hidden-state / norm traces → 长上下文位置外推 → mHC / AttnRes 深层稳定性消融</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P0</span><strong>语言模型前史二轮</strong><p>Kneser–Ney / LSTM / Bahdanau 逐图 → 真实小语料复现 → tokenizer 公平性</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
|
||||
Reference in New Issue
Block a user