wuyang
896 lines
39 KiB
Plaintext
896 lines
39 KiB
Plaintext
---
|
||
const sftTokens = [
|
||
{ text: "<system>", role: "prompt", probability: 0.94 },
|
||
{ text: "请简洁回答", role: "prompt", probability: 0.66 },
|
||
{ text: "<user>", role: "prompt", probability: 0.91 },
|
||
{ text: "为什么海是蓝色？", role: "prompt", probability: 0.42 },
|
||
{ text: "<assistant>", role: "response", probability: 0.87 },
|
||
{ text: "阳光", role: "response", probability: 0.72 },
|
||
{ text: "进入海水后", role: "response", probability: 0.58 },
|
||
{ text: "红光更易被吸收", role: "response", probability: 0.36 },
|
||
{ text: "蓝光更易散射返回", role: "response", probability: 0.29 },
|
||
{ text: "因此看起来偏蓝", role: "response", probability: 0.63 },
|
||
];
|
||
|
||
const recipes = [
|
||
{
|
||
id: "instructgpt",
|
||
label: "InstructGPT",
|
||
year: "2022",
|
||
family: "Human-feedback RLHF",
|
||
feedback: "人工 demonstration + 人工 K-way ranking",
|
||
regime: "SFT → RM → online PPO-ptx",
|
||
granularity: "Token imitation + response scalar",
|
||
models: "Policy · Reference · RM · Value",
|
||
constraints: "per-token KL + pretraining mix",
|
||
risk: "RM 过优化、标注群体边界、alignment tax",
|
||
path: ["API / 标注者 prompts", "13K SFT demos", "33K ranking prompts → RM", "31K PPO prompts", "PPO-ptx policy"],
|
||
},
|
||
{
|
||
id: "cai",
|
||
label: "Constitutional AI",
|
||
year: "2022",
|
||
family: "Principle-guided RLAIF",
|
||
feedback: "人写原则 + AI critique / revision / preference",
|
||
regime: "SL-CAI → preference model → RL-CAI",
|
||
granularity: "修订示范 + response preference",
|
||
models: "Policy · AI labeler · PM · Value",
|
||
constraints: "constitution + helpfulness objective",
|
||
risk: "原则覆盖、teacher 偏差、无害但回避",
|
||
path: ["Constitution", "Self-critique", "Revised SFT data", "AI preferences → PM", "RL-CAI"],
|
||
},
|
||
{
|
||
id: "dpo",
|
||
label: "DPO",
|
||
year: "2023",
|
||
family: "Offline direct preference",
|
||
feedback: "固定 chosen / rejected pairs",
|
||
regime: "SFT reference → offline DPO",
|
||
granularity: "sequence log-ratio pair",
|
||
models: "Policy · frozen Reference",
|
||
constraints: "β × policy/reference ratio",
|
||
risk: "pair 分布陈旧、长度口径、reference 选择",
|
||
path: ["Prompt", "Sample pairs", "Preference label", "Reference log-prob", "Direct policy update"],
|
||
},
|
||
{
|
||
id: "deepseekv2",
|
||
label: "DeepSeek-V2",
|
||
year: "2024",
|
||
family: "Multi-reward GRPO",
|
||
feedback: "人工偏好 + RM + 编译器/答案规则",
|
||
regime: "SFT → reasoning RL → preference RL",
|
||
granularity: "group-relative outcome reward",
|
||
models: "Policy · Reference · multiple RMs",
|
||
constraints: "KL + rule / helpful / safety rewards",
|
||
risk: "alignment tax、online/offline 分布、RM 可靠性",
|
||
path: ["SFT", "Reasoning RM", "GRPO stage 1", "Helpful / Safety / Rule", "GRPO stage 2"],
|
||
},
|
||
{
|
||
id: "deepseekv4",
|
||
label: "DeepSeek-V4",
|
||
year: "2026",
|
||
family: "SFT + GRPO + GRM",
|
||
feedback: "规则 verifier + rubric-guided generative reward",
|
||
regime: "SFT → unified domain GRPO",
|
||
granularity: "outcome + generated rubric score",
|
||
models: "Policy · Reference · Verifier / GRM",
|
||
constraints: "规则优先、rubric、QAT deployment constraint",
|
||
risk: "GRM 可被利用、rubric 缺口、环境覆盖不足",
|
||
path: ["Curated SFT", "Rule-verifiable tasks", "Hard tasks → GRM", "GRPO", "Domain-aligned model"],
|
||
},
|
||
{
|
||
id: "k3",
|
||
label: "Kimi K3",
|
||
year: "2026",
|
||
family: "Multi-effort expert integration",
|
||
feedback: "规则/环境/Agentic GRM + 9 teachers",
|
||
regime: "SFT → 9 RL experts → MOPD",
|
||
granularity: "outcome reward + per-token teacher signal",
|
||
models: "Student · 9 Teachers · Verifiers / GRM",
|
||
constraints: "partial-rollout clipping + budget + verbosity",
|
||
risk: "长轨迹陈旧、teacher 偏差、reward hacking",
|
||
path: ["Cold-start SFT", "3 domains × 3 efforts", "Partial-rollout RL", "Student on-policy rollout", "MOPD consolidation"],
|
||
},
|
||
];
|
||
---
|
||
|
||
<figure
|
||
class="alignment-lab"
|
||
data-alignment-lab
|
||
data-sft-tokens={JSON.stringify(sftTokens)}
|
||
data-recipes={JSON.stringify(recipes)}
|
||
>
|
||
<div class="lab-heading">
|
||
<div>
|
||
<span class="micro-label">INTERACTIVE / FOUR-VIEW LAB</span>
|
||
<h3>把“对齐”拆成四个不相互冒充的实验</h3>
|
||
</div>
|
||
<p>所有数字都是教学模拟，用来展示目标函数和反馈结构；不是任何模型的训练日志或产品指标。</p>
|
||
</div>
|
||
|
||
<div class="lab-tabs" role="tablist" aria-label="后训练四联实验">
|
||
<button type="button" role="tab" aria-selected="true" aria-controls="alignment-panel-sft" id="alignment-tab-sft" data-alignment-tab="sft">
|
||
<span>01</span><b>SFT 显微镜</b><small>哪些 Token 产生梯度</small>
|
||
</button>
|
||
<button type="button" role="tab" aria-selected="false" aria-controls="alignment-panel-preference" id="alignment-tab-preference" data-alignment-tab="preference" tabindex="-1">
|
||
<span>02</span><b>偏好与 RM</b><small>比较怎样变成标量</small>
|
||
</button>
|
||
<button type="button" role="tab" aria-selected="false" aria-controls="alignment-panel-update" id="alignment-tab-update" data-alignment-tab="update" tabindex="-1">
|
||
<span>03</span><b>PPO / DPO</b><small>两种更新合同</small>
|
||
</button>
|
||
<button type="button" role="tab" aria-selected="false" aria-controls="alignment-panel-recipe" id="alignment-tab-recipe" data-alignment-tab="recipe" tabindex="-1">
|
||
<span>04</span><b>现代配方</b><small>2022 → K3</small>
|
||
</button>
|
||
</div>
|
||
|
||
<section class="lab-panel" role="tabpanel" id="alignment-panel-sft" aria-labelledby="alignment-tab-sft" data-alignment-panel="sft">
|
||
<div class="panel-intro">
|
||
<div><span>QUESTION</span><h4>SFT 到底在模仿 prompt，还是只模仿 assistant？</h4></div>
|
||
<p>真实 chat template 会把角色标记也编码成 Token；是否计入 loss 由 mask 决定，而不是由颜色决定。</p>
|
||
</div>
|
||
|
||
<div class="sft-controls">
|
||
<div class="mode-switch" role="group" aria-label="选择 SFT loss mask">
|
||
<button type="button" class="active" data-sft-mask="response">只训练 Assistant</button>
|
||
<button type="button" data-sft-mask="all">全序列训练</button>
|
||
</div>
|
||
<label>
|
||
Demonstration 质量
|
||
<select data-sft-quality>
|
||
<option value="expert" selected>专家、简洁、事实正确</option>
|
||
<option value="verbose">基本正确，但冗长</option>
|
||
<option value="unsafe">格式流畅，但事实错误</option>
|
||
</select>
|
||
</label>
|
||
</div>
|
||
|
||
<div class="sft-sequence" data-sft-sequence>
|
||
{sftTokens.map((token, index) => (
|
||
<div data-sft-token={index} data-role={token.role}>
|
||
<span>{token.role === "prompt" ? "CONDITION" : "TARGET"}</span>
|
||
<b>{token.text}</b>
|
||
<i><em data-sft-prob={index}>{Math.round(token.probability * 100)}%</em><small data-sft-loss={index}>—</small></i>
|
||
</div>
|
||
))}
|
||
</div>
|
||
|
||
<div class="metric-row">
|
||
<article><span>LOSS TOKENS</span><b data-sft-active>6 / 10</b><p>真正进入交叉熵求和的 Token</p></article>
|
||
<article><span>MEAN NLL</span><b data-sft-nll>0.72</b><p>只反映这条示范的拟合难度</p></article>
|
||
<article><span>TEACHER FORCING</span><b>ON</b><p>第 t 步总能看到参考答案的前缀</p></article>
|
||
</div>
|
||
|
||
<div class="lab-takeaway">
|
||
<b data-sft-reading>SFT 正在学习一条高质量示范，但它没有直接比较其他可能回答。</b>
|
||
<p>如果 demonstration 本身错误，交叉熵仍会忠实提高错误 Token 的概率；loss 并不知道“事实正确”。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="lab-panel" role="tabpanel" id="alignment-panel-preference" aria-labelledby="alignment-tab-preference" data-alignment-panel="preference" hidden>
|
||
<div class="panel-intro">
|
||
<div><span>QUESTION</span><h4>为什么“chosen”只是相对更好，不一定是好答案？</h4></div>
|
||
<p>滑块控制的是一个教学 reward model 的偏好权重；真实 RM 会从标注数据中学习这些权重与 shortcut。</p>
|
||
</div>
|
||
|
||
<div class="preference-presets" role="group" aria-label="选择偏好场景">
|
||
<button type="button" class="active" data-pref-preset="evidence">事实胜过语气</button>
|
||
<button type="button" data-pref-preset="length">长度 Shortcut</button>
|
||
<button type="button" data-pref-preset="bothbad">两个都不好</button>
|
||
<button type="button" data-pref-preset="disagree">标注者分歧</button>
|
||
</div>
|
||
|
||
<div class="preference-grid">
|
||
<article data-answer-card="a">
|
||
<span>RESPONSE A</span>
|
||
<h4 data-answer-title="a">短而有证据</h4>
|
||
<p data-answer-copy="a">指出海水对长波吸收更强，并说明散射与反射共同影响观察颜色。</p>
|
||
<dl>
|
||
<div><dt>事实</dt><dd data-feature="a-fact">92</dd></div>
|
||
<div><dt>帮助</dt><dd data-feature="a-help">78</dd></div>
|
||
<div><dt>风格</dt><dd data-feature="a-style">58</dd></div>
|
||
<div><dt>长度</dt><dd data-feature="a-length">34</dd></div>
|
||
</dl>
|
||
<strong data-reward="a">+1.42</strong>
|
||
</article>
|
||
<div class="versus">
|
||
<span>BRADLEY–TERRY</span>
|
||
<b data-pref-probability>74%</b>
|
||
<small data-pref-choice>P(A ≻ B)</small>
|
||
<i><em data-pref-meter></em></i>
|
||
</div>
|
||
<article data-answer-card="b">
|
||
<span>RESPONSE B</span>
|
||
<h4 data-answer-title="b">流畅但错误</h4>
|
||
<p data-answer-copy="b">写得更长、更有自信，却把海水颜色完全归因于天空倒影。</p>
|
||
<dl>
|
||
<div><dt>事实</dt><dd data-feature="b-fact">24</dd></div>
|
||
<div><dt>帮助</dt><dd data-feature="b-help">46</dd></div>
|
||
<div><dt>风格</dt><dd data-feature="b-style">86</dd></div>
|
||
<div><dt>长度</dt><dd data-feature="b-length">78</dd></div>
|
||
</dl>
|
||
<strong data-reward="b">+0.39</strong>
|
||
</article>
|
||
</div>
|
||
|
||
<div class="rm-controls">
|
||
<label>事实权重<input type="range" min="0" max="100" value="70" data-rm-weight="fact" /><output data-rm-label="fact">0.70</output></label>
|
||
<label>帮助权重<input type="range" min="0" max="100" value="50" data-rm-weight="help" /><output data-rm-label="help">0.50</output></label>
|
||
<label>风格权重<input type="range" min="0" max="100" value="25" data-rm-weight="style" /><output data-rm-label="style">0.25</output></label>
|
||
<label>长度权重<input type="range" min="-50" max="80" value="5" data-rm-weight="length" /><output data-rm-label="length">0.05</output></label>
|
||
<label>标签噪声<input type="range" min="0" max="45" value="8" data-rm-noise /><output data-rm-noise-label>8%</output></label>
|
||
</div>
|
||
|
||
<div class="lab-takeaway">
|
||
<b data-pref-reading>这个 RM 更看重事实，因此短回答 A 获得更高偏好概率。</b>
|
||
<p>概率来自 reward difference，不是绝对质量证书；如果两个回答都很差，模型仍会被要求选出一个相对赢家。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="lab-panel" role="tabpanel" id="alignment-panel-update" aria-labelledby="alignment-tab-update" data-alignment-panel="update" hidden>
|
||
<div class="panel-intro">
|
||
<div><span>QUESTION</span><h4>PPO 与 DPO 到底在哪一步分叉？</h4></div>
|
||
<p>二者都可从 pairwise preference 出发，但 PPO 显式拟合 RM 并向当前 policy 采样；经典 DPO 直接消费固定 pairs。</p>
|
||
</div>
|
||
|
||
<div class="update-switch" role="group" aria-label="选择偏好优化方法">
|
||
<button type="button" class="active" data-update-method="ppo">PPO / explicit RM</button>
|
||
<button type="button" data-update-method="dpo">DPO / implicit RM</button>
|
||
</div>
|
||
|
||
<div class="update-workbench">
|
||
<div class="update-flow" data-update-flow>
|
||
<div><span>01</span><b data-update-step="0">Current policy rollout</b></div>
|
||
<i>→</i>
|
||
<div><span>02</span><b data-update-step="1">Reward model score</b></div>
|
||
<i>→</i>
|
||
<div><span>03</span><b data-update-step="2">Advantage + critic</b></div>
|
||
<i>→</i>
|
||
<div><span>04</span><b data-update-step="3">Clipped policy update</b></div>
|
||
</div>
|
||
|
||
<div class="update-controls">
|
||
<label>
|
||
Reward / preference gap
|
||
<input type="range" min="-100" max="200" value="80" data-update-gap />
|
||
<output data-update-gap-label>+0.80</output>
|
||
</label>
|
||
<label>
|
||
KL / β 约束
|
||
<input type="range" min="1" max="100" value="25" data-update-beta />
|
||
<output data-update-beta-label>0.25</output>
|
||
</label>
|
||
<label>
|
||
Policy 相对 reference 位移
|
||
<input type="range" min="-100" max="160" value="40" data-update-shift />
|
||
<output data-update-shift-label>+0.40</output>
|
||
</label>
|
||
<label>
|
||
Pair / rollout 陈旧度
|
||
<input type="range" min="0" max="100" value="15" data-update-stale />
|
||
<output data-update-stale-label>15%</output>
|
||
</label>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="metric-row">
|
||
<article><span>TRAINING SIGNAL</span><b data-update-signal>+0.41</b><p data-update-signal-note>RM advantage 经 PPO clip 与 KL 后的教学量</p></article>
|
||
<article><span>LIVE MODELS</span><b data-update-models>4</b><p data-update-model-note>policy · reference · RM · value</p></article>
|
||
<article><span>DATA REGIME</span><b data-update-regime>ONLINE</b><p data-update-regime-note>需要接近当前 policy 的 rollout</p></article>
|
||
</div>
|
||
|
||
<div class="equation-card">
|
||
<span data-update-kicker>PPO SKETCH</span>
|
||
<code data-update-equation>min(ρA, clip(ρ)A) − β KL(π || πref)</code>
|
||
<p data-update-reading>当 policy 已明显偏离 behavior policy，importance ratio 与 critic 误差会让更新更脆弱。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="lab-panel" role="tabpanel" id="alignment-panel-recipe" aria-labelledby="alignment-tab-recipe" data-alignment-panel="recipe" hidden>
|
||
<div class="panel-intro">
|
||
<div><span>QUESTION</span><h4>现代 post-training 为什么不再是一条固定的 SFT→RM→PPO 流水线？</h4></div>
|
||
<p>用同一组字段比较反馈来源、训练制度、监督粒度、约束与失效模式，避免只比较算法缩写。</p>
|
||
</div>
|
||
|
||
<div class="recipe-selector" role="group" aria-label="选择 post-training 配方">
|
||
{recipes.map((recipe, index) => (
|
||
<button type="button" class:list={{ active: index === 0 }} data-recipe-id={recipe.id}>
|
||
<span>{recipe.year}</span>{recipe.label}
|
||
</button>
|
||
))}
|
||
</div>
|
||
|
||
<div class="recipe-layout">
|
||
<div class="recipe-path" data-recipe-path>
|
||
{recipes[0].path.map((step, index) => (
|
||
<div>
|
||
<span>{String(index + 1).padStart(2, "0")}</span>
|
||
<b data-recipe-step>{step}</b>
|
||
</div>
|
||
))}
|
||
</div>
|
||
<dl class="recipe-contract">
|
||
<div><dt>FAMILY</dt><dd data-recipe-family>{recipes[0].family}</dd></div>
|
||
<div><dt>FEEDBACK</dt><dd data-recipe-feedback>{recipes[0].feedback}</dd></div>
|
||
<div><dt>REGIME</dt><dd data-recipe-regime>{recipes[0].regime}</dd></div>
|
||
<div><dt>GRANULARITY</dt><dd data-recipe-granularity>{recipes[0].granularity}</dd></div>
|
||
<div><dt>LIVE MODELS</dt><dd data-recipe-models>{recipes[0].models}</dd></div>
|
||
<div><dt>CONSTRAINTS</dt><dd data-recipe-constraints>{recipes[0].constraints}</dd></div>
|
||
</dl>
|
||
</div>
|
||
|
||
<div class="risk-strip">
|
||
<span>PRIMARY FAILURE MODE</span>
|
||
<b data-recipe-risk>{recipes[0].risk}</b>
|
||
<p>这是一张角色合同，不是排行榜；不同配方面对的任务、数据与预算并不相同。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<figcaption>
|
||
<b>交互图 10</b>
|
||
四个实验依次回答：示范怎样产生 Token 梯度、偏好怎样变成代理标量、policy 怎样更新，以及现代配方怎样组合多种反馈。
|
||
</figcaption>
|
||
</figure>
|
||
|
||
<script>
|
||
type SftToken = { text: string; role: "prompt" | "response"; probability: number };
|
||
type Recipe = {
|
||
id: string;
|
||
label: string;
|
||
year: string;
|
||
family: string;
|
||
feedback: string;
|
||
regime: string;
|
||
granularity: string;
|
||
models: string;
|
||
constraints: string;
|
||
risk: string;
|
||
path: string[];
|
||
};
|
||
|
||
document.querySelectorAll<HTMLElement>("[data-alignment-lab]").forEach((root) => {
|
||
const tabs = [...root.querySelectorAll<HTMLButtonElement>("[data-alignment-tab]")];
|
||
const panels = [...root.querySelectorAll<HTMLElement>("[data-alignment-panel]")];
|
||
const sftTokens = JSON.parse(root.dataset.sftTokens ?? "[]") as SftToken[];
|
||
const recipes = JSON.parse(root.dataset.recipes ?? "[]") as Recipe[];
|
||
|
||
const setText = (selector: string, text: string) => {
|
||
const node = root.querySelector<HTMLElement>(selector);
|
||
if (node) node.textContent = text;
|
||
};
|
||
const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value));
|
||
const sigmoid = (value: number) => 1 / (1 + Math.exp(-value));
|
||
|
||
const activateTab = (name: string, focus = false) => {
|
||
tabs.forEach((tab) => {
|
||
const active = tab.dataset.alignmentTab === name;
|
||
tab.setAttribute("aria-selected", String(active));
|
||
tab.tabIndex = active ? 0 : -1;
|
||
if (active && focus) tab.focus();
|
||
});
|
||
panels.forEach((panel) => {
|
||
panel.hidden = panel.dataset.alignmentPanel !== name;
|
||
});
|
||
};
|
||
tabs.forEach((tab, index) => {
|
||
tab.addEventListener("click", () => activateTab(tab.dataset.alignmentTab ?? "sft"));
|
||
tab.addEventListener("keydown", (event) => {
|
||
let next = index;
|
||
if (event.key === "ArrowRight") next = (index + 1) % tabs.length;
|
||
else if (event.key === "ArrowLeft") next = (index - 1 + tabs.length) % tabs.length;
|
||
else if (event.key === "Home") next = 0;
|
||
else if (event.key === "End") next = tabs.length - 1;
|
||
else return;
|
||
event.preventDefault();
|
||
activateTab(tabs[next].dataset.alignmentTab ?? "sft", true);
|
||
});
|
||
});
|
||
|
||
let sftMask = "response";
|
||
const qualitySelect = root.querySelector<HTMLSelectElement>("[data-sft-quality]");
|
||
const qualityMultipliers: Record<string, { probabilities: number[]; reading: string }> = {
|
||
expert: {
|
||
probabilities: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
|
||
reading: "SFT 正在学习一条高质量示范，但它没有直接比较其他可能回答。",
|
||
},
|
||
verbose: {
|
||
probabilities: [1, 1, 1, 1, 1, 0.92, 0.84, 0.78, 0.72, 0.68],
|
||
reading: "冗长示范提供更多稠密梯度，也会把重复、铺垫和表达习惯一起写进 policy。",
|
||
},
|
||
unsafe: {
|
||
probabilities: [1, 1, 1, 1, 1, 0.7, 0.46, 0.3, 0.24, 0.34],
|
||
reading: "交叉熵不会识别事实错误：只要它是 target，错误回答同样会得到正向模仿梯度。",
|
||
},
|
||
};
|
||
const renderSft = () => {
|
||
const quality = qualitySelect?.value ?? "expert";
|
||
const profile = qualityMultipliers[quality];
|
||
let active = 0;
|
||
let totalNll = 0;
|
||
sftTokens.forEach((token, index) => {
|
||
const contributes = sftMask === "all" || token.role === "response";
|
||
const probability = clamp(token.probability * (profile.probabilities[index] ?? 1), 0.04, 0.99);
|
||
const loss = -Math.log(probability);
|
||
const card = root.querySelector<HTMLElement>(`[data-sft-token="${index}"]`);
|
||
card?.toggleAttribute("data-active", contributes);
|
||
setText(`[data-sft-prob="${index}"]`, `${Math.round(probability * 100)}%`);
|
||
setText(`[data-sft-loss="${index}"]`, contributes ? `NLL ${loss.toFixed(2)}` : "MASKED");
|
||
if (contributes) {
|
||
active += 1;
|
||
totalNll += loss;
|
||
}
|
||
});
|
||
setText("[data-sft-active]", `${active} / ${sftTokens.length}`);
|
||
setText("[data-sft-nll]", (totalNll / Math.max(active, 1)).toFixed(2));
|
||
setText("[data-sft-reading]", profile.reading);
|
||
root.querySelectorAll<HTMLButtonElement>("[data-sft-mask]").forEach((button) => {
|
||
button.classList.toggle("active", button.dataset.sftMask === sftMask);
|
||
});
|
||
};
|
||
root.querySelectorAll<HTMLButtonElement>("[data-sft-mask]").forEach((button) => {
|
||
button.addEventListener("click", () => {
|
||
sftMask = button.dataset.sftMask ?? "response";
|
||
renderSft();
|
||
});
|
||
});
|
||
qualitySelect?.addEventListener("change", renderSft);
|
||
renderSft();
|
||
|
||
type Answer = { title: string; copy: string; fact: number; help: number; style: number; length: number };
|
||
const preferenceProfiles: Record<string, { a: Answer; b: Answer; weights: number[]; noise: number; reading: string }> = {
|
||
evidence: {
|
||
a: { title: "短而有证据", copy: "指出海水对长波吸收更强，并说明散射与反射共同影响观察颜色。", fact: 92, help: 78, style: 58, length: 34 },
|
||
b: { title: "流畅但错误", copy: "写得更长、更有自信，却把海水颜色完全归因于天空倒影。", fact: 24, help: 46, style: 86, length: 78 },
|
||
weights: [70, 50, 25, 5], noise: 8,
|
||
reading: "这个 RM 更看重事实，因此短回答 A 获得更高偏好概率。",
|
||
},
|
||
length: {
|
||
a: { title: "短而正确", copy: "答案正确、简洁，但没有铺陈背景或重复结论。", fact: 90, help: 72, style: 52, length: 24 },
|
||
b: { title: "长而空泛", copy: "分成很多段并反复总结，信息增量有限，但表面上显得完整。", fact: 54, help: 58, style: 78, length: 96 },
|
||
weights: [35, 35, 35, 65], noise: 10,
|
||
reading: "长度被赋予过高权重时，RM 会把“写得多”误当成“帮助更大”。",
|
||
},
|
||
bothbad: {
|
||
a: { title: "简短猜测", copy: "给出一个没有证据的猜测，遗漏关键机制。", fact: 28, help: 24, style: 42, length: 20 },
|
||
b: { title: "自信编造", copy: "用流畅长文包装错误因果，还加入不存在的数字。", fact: 12, help: 30, style: 76, length: 82 },
|
||
weights: [60, 45, 30, 5], noise: 12,
|
||
reading: "A 只是“两坏相较取其轻”；chosen 标签不能把它升级成高质量 demonstration。",
|
||
},
|
||
disagree: {
|
||
a: { title: "直接纠错", copy: "明确指出用户前提错误，事实准确但语气生硬。", fact: 94, help: 62, style: 28, length: 38 },
|
||
b: { title: "温和迎合", copy: "先认同用户，再给模糊建议；更悦耳但保留错误前提。", fact: 42, help: 66, style: 91, length: 58 },
|
||
weights: [48, 50, 55, 8], noise: 38,
|
||
reading: "当标注者在“直接事实”与“关系体验”之间分歧，单一标量会把差异压成平均偏好。",
|
||
},
|
||
};
|
||
let currentPreference = preferenceProfiles.evidence;
|
||
const weightInputs = [...root.querySelectorAll<HTMLInputElement>("[data-rm-weight]")];
|
||
const noiseInput = root.querySelector<HTMLInputElement>("[data-rm-noise]");
|
||
const updateAnswer = (side: "a" | "b", answer: Answer) => {
|
||
setText(`[data-answer-title="${side}"]`, answer.title);
|
||
setText(`[data-answer-copy="${side}"]`, answer.copy);
|
||
(["fact", "help", "style", "length"] as const).forEach((feature) => {
|
||
setText(`[data-feature="${side}-${feature}"]`, String(answer[feature]));
|
||
});
|
||
};
|
||
const renderPreference = () => {
|
||
const weights = Object.fromEntries(weightInputs.map((input) => [input.dataset.rmWeight ?? "", Number(input.value) / 100])) as Record<string, number>;
|
||
const noise = Number(noiseInput?.value ?? 0) / 100;
|
||
const score = (answer: Answer) => (
|
||
answer.fact * weights.fact +
|
||
answer.help * weights.help +
|
||
answer.style * weights.style +
|
||
answer.length * weights.length
|
||
) / 100 - 0.95;
|
||
const rewardA = score(currentPreference.a);
|
||
const rewardB = score(currentPreference.b);
|
||
const cleanProbability = sigmoid(rewardA - rewardB);
|
||
const probability = cleanProbability * (1 - noise) + 0.5 * noise;
|
||
setText('[data-reward="a"]', `${rewardA >= 0 ? "+" : ""}${rewardA.toFixed(2)}`);
|
||
setText('[data-reward="b"]', `${rewardB >= 0 ? "+" : ""}${rewardB.toFixed(2)}`);
|
||
setText("[data-pref-probability]", `${Math.round(probability * 100)}%`);
|
||
setText("[data-pref-choice]", probability >= 0.5 ? "P(A ≻ B)" : "P(B ≻ A)");
|
||
const meter = root.querySelector<HTMLElement>("[data-pref-meter]");
|
||
if (meter) meter.style.width = `${probability * 100}%`;
|
||
weightInputs.forEach((input) => {
|
||
const key = input.dataset.rmWeight ?? "";
|
||
setText(`[data-rm-label="${key}"]`, (Number(input.value) / 100).toFixed(2));
|
||
});
|
||
setText("[data-rm-noise-label]", `${Math.round(noise * 100)}%`);
|
||
setText("[data-pref-reading]", currentPreference.reading);
|
||
};
|
||
root.querySelectorAll<HTMLButtonElement>("[data-pref-preset]").forEach((button) => {
|
||
button.addEventListener("click", () => {
|
||
currentPreference = preferenceProfiles[button.dataset.prefPreset ?? "evidence"];
|
||
updateAnswer("a", currentPreference.a);
|
||
updateAnswer("b", currentPreference.b);
|
||
weightInputs.forEach((input, index) => { input.value = String(currentPreference.weights[index]); });
|
||
if (noiseInput) noiseInput.value = String(currentPreference.noise);
|
||
root.querySelectorAll<HTMLButtonElement>("[data-pref-preset]").forEach((item) => item.classList.toggle("active", item === button));
|
||
renderPreference();
|
||
});
|
||
});
|
||
[...weightInputs, noiseInput].forEach((input) => input?.addEventListener("input", renderPreference));
|
||
renderPreference();
|
||
|
||
let updateMethod = "ppo";
|
||
const gapInput = root.querySelector<HTMLInputElement>("[data-update-gap]");
|
||
const betaInput = root.querySelector<HTMLInputElement>("[data-update-beta]");
|
||
const shiftInput = root.querySelector<HTMLInputElement>("[data-update-shift]");
|
||
const staleInput = root.querySelector<HTMLInputElement>("[data-update-stale]");
|
||
const renderUpdate = () => {
|
||
const gap = Number(gapInput?.value ?? 80) / 100;
|
||
const beta = Number(betaInput?.value ?? 25) / 100;
|
||
const shift = Number(shiftInput?.value ?? 40) / 100;
|
||
const stale = Number(staleInput?.value ?? 15) / 100;
|
||
setText("[data-update-gap-label]", `${gap >= 0 ? "+" : ""}${gap.toFixed(2)}`);
|
||
setText("[data-update-beta-label]", beta.toFixed(2));
|
||
setText("[data-update-shift-label]", `${shift >= 0 ? "+" : ""}${shift.toFixed(2)}`);
|
||
setText("[data-update-stale-label]", `${Math.round(stale * 100)}%`);
|
||
root.querySelectorAll<HTMLButtonElement>("[data-update-method]").forEach((button) => {
|
||
button.classList.toggle("active", button.dataset.updateMethod === updateMethod);
|
||
});
|
||
if (updateMethod === "ppo") {
|
||
const ratio = Math.exp(shift * (1 + stale));
|
||
const clipped = clamp(ratio, 0.8, 1.2);
|
||
const signal = Math.min(ratio * gap, clipped * gap) - beta * Math.abs(shift);
|
||
["Current policy rollout", "Reward model score", "Advantage + critic", "Clipped policy update"].forEach((step, index) => setText(`[data-update-step="${index}"]`, step));
|
||
setText("[data-update-signal]", `${signal >= 0 ? "+" : ""}${signal.toFixed(2)}`);
|
||
setText("[data-update-signal-note]", "RM advantage 经 PPO clip 与 KL 后的教学量");
|
||
setText("[data-update-models]", "4");
|
||
setText("[data-update-model-note]", "policy · reference · RM · value");
|
||
setText("[data-update-regime]", "ONLINE");
|
||
setText("[data-update-regime-note]", "需要接近当前 policy 的 rollout");
|
||
setText("[data-update-kicker]", "PPO SKETCH");
|
||
setText("[data-update-equation]", "min(ρA, clip(ρ)A) − β KL(π || πref)");
|
||
setText("[data-update-reading]", stale > 0.55
|
||
? "rollout 已明显陈旧；importance ratio、critic 与长轨迹 credit 会让更新更脆弱。"
|
||
: "PPO 显式消费 rollout 与 scalar reward；clip 约束单次 policy 位移，但不修复错误奖励。");
|
||
} else {
|
||
const implicitMargin = beta * (gap - shift * stale);
|
||
const loss = -Math.log(sigmoid(implicitMargin));
|
||
["Fixed preference pairs", "Policy / reference log-probs", "Implicit reward margin", "Classification-style update"].forEach((step, index) => setText(`[data-update-step="${index}"]`, step));
|
||
setText("[data-update-signal]", implicitMargin.toFixed(2));
|
||
setText("[data-update-signal-note]", "β × chosen/rejected 的相对 log-ratio margin");
|
||
setText("[data-update-models]", "2");
|
||
setText("[data-update-model-note]", "trainable policy · frozen reference");
|
||
setText("[data-update-regime]", "OFFLINE");
|
||
setText("[data-update-regime-note]", "经典 DPO 不在训练中向当前 policy 采样");
|
||
setText("[data-update-kicker]", "DPO SKETCH");
|
||
setText("[data-update-equation]", `−log σ(β Δlog(π/πref)) = ${loss.toFixed(2)}`);
|
||
setText("[data-update-reading]", stale > 0.55
|
||
? "pair 与当前 policy 差距很大：目标仍能计算，但它回答的是旧候选分布上的偏好。"
|
||
: "DPO 省去显式 RM 与在线 RL loop；偏好 pair、reference、β 和分布匹配仍然存在。");
|
||
}
|
||
};
|
||
root.querySelectorAll<HTMLButtonElement>("[data-update-method]").forEach((button) => {
|
||
button.addEventListener("click", () => {
|
||
updateMethod = button.dataset.updateMethod ?? "ppo";
|
||
renderUpdate();
|
||
});
|
||
});
|
||
[gapInput, betaInput, shiftInput, staleInput].forEach((input) => input?.addEventListener("input", renderUpdate));
|
||
renderUpdate();
|
||
|
||
const recipeButtons = [...root.querySelectorAll<HTMLButtonElement>("[data-recipe-id]")];
|
||
const renderRecipe = (id: string) => {
|
||
const recipe = recipes.find((item) => item.id === id) ?? recipes[0];
|
||
recipeButtons.forEach((button) => button.classList.toggle("active", button.dataset.recipeId === recipe.id));
|
||
const stepNodes = [...root.querySelectorAll<HTMLElement>("[data-recipe-step]")];
|
||
stepNodes.forEach((node, index) => { node.textContent = recipe.path[index] ?? ""; });
|
||
(["family", "feedback", "regime", "granularity", "models", "constraints", "risk"] as const).forEach((field) => {
|
||
setText(`[data-recipe-${field}]`, recipe[field]);
|
||
});
|
||
};
|
||
recipeButtons.forEach((button) => button.addEventListener("click", () => renderRecipe(button.dataset.recipeId ?? "instructgpt")));
|
||
renderRecipe("instructgpt");
|
||
});
|
||
</script>
|
||
|
||
<style>
|
||
.alignment-lab {
|
||
max-width: 1020px;
|
||
margin: 42px 0;
|
||
border: 1px solid var(--line);
|
||
background: var(--paper-raised);
|
||
}
|
||
|
||
.lab-heading,
|
||
.panel-intro {
|
||
display: grid;
|
||
grid-template-columns: 1fr minmax(250px, 350px);
|
||
gap: 34px;
|
||
align-items: end;
|
||
}
|
||
|
||
.lab-heading {
|
||
padding: 30px;
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.lab-heading h3 {
|
||
margin-top: 10px;
|
||
font-size: 1.55rem;
|
||
}
|
||
|
||
.lab-heading p,
|
||
.panel-intro p {
|
||
color: var(--muted);
|
||
font-size: .72rem;
|
||
line-height: 1.75;
|
||
}
|
||
|
||
.lab-tabs {
|
||
display: grid;
|
||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.lab-tabs button {
|
||
display: grid;
|
||
gap: 7px;
|
||
min-height: 92px;
|
||
padding: 15px 17px;
|
||
border: 0;
|
||
border-right: 1px solid var(--line);
|
||
background: transparent;
|
||
color: var(--muted);
|
||
text-align: left;
|
||
}
|
||
|
||
.lab-tabs button:last-child { border-right: 0; }
|
||
.lab-tabs button[aria-selected="true"] { color: white; background: var(--navy); }
|
||
.lab-tabs span,
|
||
.panel-intro span,
|
||
.preference-grid > article > span,
|
||
.versus span,
|
||
.risk-strip span,
|
||
.equation-card span {
|
||
color: var(--copper);
|
||
font: .58rem/1.2 var(--mono);
|
||
letter-spacing: .09em;
|
||
}
|
||
.lab-tabs b { font: 600 .85rem/1.15 var(--sans); }
|
||
.lab-tabs small { font: .54rem/1.2 var(--mono); }
|
||
|
||
.lab-panel { padding: 30px; }
|
||
.panel-intro { margin-bottom: 28px; }
|
||
.panel-intro h4 { margin-top: 8px; font-size: 1.28rem; }
|
||
|
||
.mode-switch,
|
||
.preference-presets,
|
||
.update-switch,
|
||
.recipe-selector {
|
||
display: flex;
|
||
flex-wrap: wrap;
|
||
gap: 8px;
|
||
}
|
||
|
||
.mode-switch button,
|
||
.preference-presets button,
|
||
.update-switch button,
|
||
.recipe-selector button {
|
||
padding: 10px 13px;
|
||
border: 1px solid var(--line);
|
||
background: var(--paper);
|
||
color: var(--muted);
|
||
font: .62rem var(--mono);
|
||
}
|
||
|
||
.mode-switch button.active,
|
||
.preference-presets button.active,
|
||
.update-switch button.active,
|
||
.recipe-selector button.active {
|
||
border-color: var(--navy);
|
||
background: var(--navy);
|
||
color: white;
|
||
}
|
||
|
||
.sft-controls {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
gap: 20px;
|
||
align-items: center;
|
||
}
|
||
|
||
.sft-controls label,
|
||
.update-controls label,
|
||
.rm-controls label {
|
||
color: var(--muted);
|
||
font: .62rem var(--mono);
|
||
}
|
||
|
||
select {
|
||
min-height: 38px;
|
||
margin-left: 8px;
|
||
border: 1px solid var(--line);
|
||
background: var(--paper);
|
||
color: var(--ink);
|
||
}
|
||
|
||
.sft-sequence {
|
||
display: grid;
|
||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||
gap: 1px;
|
||
margin-top: 24px;
|
||
border: 1px solid var(--line);
|
||
background: var(--line);
|
||
}
|
||
|
||
.sft-sequence > div {
|
||
min-height: 126px;
|
||
padding: 13px;
|
||
background: var(--paper);
|
||
opacity: .56;
|
||
}
|
||
.sft-sequence > div[data-active] { opacity: 1; box-shadow: inset 0 -3px var(--copper); }
|
||
.sft-sequence > div[data-role="response"] { background: #f0eadf; }
|
||
.sft-sequence span { color: var(--muted); font: .5rem var(--mono); }
|
||
.sft-sequence b { display: block; min-height: 42px; margin: 12px 0; font-size: .7rem; }
|
||
.sft-sequence i { display: flex; justify-content: space-between; gap: 6px; font-style: normal; }
|
||
.sft-sequence em { color: var(--navy); font: normal 600 .62rem var(--mono); }
|
||
.sft-sequence small { color: var(--copper); font: .52rem var(--mono); }
|
||
|
||
.metric-row {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
gap: 1px;
|
||
margin-top: 24px;
|
||
border: 1px solid var(--line);
|
||
background: var(--line);
|
||
}
|
||
.metric-row article { padding: 18px; background: var(--paper); }
|
||
.metric-row span { color: var(--muted); font: .52rem var(--mono); }
|
||
.metric-row b { display: block; margin: 11px 0 6px; color: var(--navy); font: 600 1.25rem var(--serif); }
|
||
.metric-row p { margin: 0; color: var(--muted); font-size: .62rem; }
|
||
|
||
.lab-takeaway {
|
||
margin-top: 20px;
|
||
padding: 18px 20px;
|
||
border-left: 3px solid var(--copper);
|
||
background: #f1eadf;
|
||
}
|
||
.lab-takeaway b { color: var(--navy); }
|
||
.lab-takeaway p { margin: 8px 0 0; color: var(--muted); font-size: .7rem; }
|
||
|
||
.preference-grid {
|
||
display: grid;
|
||
grid-template-columns: 1fr 120px 1fr;
|
||
gap: 1px;
|
||
margin-top: 22px;
|
||
border: 1px solid var(--line);
|
||
background: var(--line);
|
||
}
|
||
.preference-grid > article { padding: 20px; background: var(--paper); }
|
||
.preference-grid h4 { margin: 10px 0; font-size: 1rem; }
|
||
.preference-grid p { min-height: 64px; color: var(--muted); font-size: .68rem; }
|
||
.preference-grid dl { display: grid; grid-template-columns: repeat(4, 1fr); gap: 1px; margin: 18px 0; background: var(--line); }
|
||
.preference-grid dl div { padding: 9px 5px; background: white; text-align: center; }
|
||
.preference-grid dt { color: var(--muted); font: .48rem var(--mono); }
|
||
.preference-grid dd { margin: 6px 0 0; font: 600 .72rem var(--mono); }
|
||
.preference-grid article > strong { color: var(--navy); font: 600 1.35rem var(--serif); }
|
||
|
||
.versus {
|
||
display: flex;
|
||
flex-direction: column;
|
||
align-items: center;
|
||
justify-content: center;
|
||
gap: 10px;
|
||
padding: 12px;
|
||
background: var(--navy);
|
||
color: white;
|
||
text-align: center;
|
||
}
|
||
.versus b { font: 600 1.6rem var(--serif); }
|
||
.versus small { font: .5rem var(--mono); }
|
||
.versus i { width: 100%; height: 5px; background: #526477; }
|
||
.versus em { display: block; height: 100%; background: var(--copper); }
|
||
|
||
.rm-controls,
|
||
.update-controls {
|
||
display: grid;
|
||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||
gap: 14px;
|
||
margin-top: 20px;
|
||
padding: 18px;
|
||
border: 1px solid var(--line);
|
||
background: #f7f3eb;
|
||
}
|
||
.rm-controls label,
|
||
.update-controls label { display: grid; gap: 8px; }
|
||
input[type="range"] { width: 100%; accent-color: var(--copper); }
|
||
output { color: var(--navy); font-weight: 700; }
|
||
|
||
.update-workbench {
|
||
display: grid;
|
||
grid-template-columns: 1fr 300px;
|
||
gap: 22px;
|
||
margin-top: 22px;
|
||
}
|
||
.update-flow {
|
||
display: flex;
|
||
align-items: stretch;
|
||
gap: 8px;
|
||
}
|
||
.update-flow div {
|
||
display: grid;
|
||
flex: 1;
|
||
align-content: center;
|
||
gap: 8px;
|
||
min-height: 124px;
|
||
padding: 14px;
|
||
border: 1px solid var(--line);
|
||
background: var(--paper);
|
||
}
|
||
.update-flow span,
|
||
.recipe-path span { color: var(--copper); font: .55rem var(--mono); }
|
||
.update-flow b { font-size: .67rem; }
|
||
.update-flow i { align-self: center; color: var(--copper); font-style: normal; }
|
||
.update-controls { grid-template-columns: 1fr; margin: 0; }
|
||
|
||
.equation-card {
|
||
margin-top: 20px;
|
||
padding: 20px;
|
||
background: #25364b;
|
||
color: white;
|
||
}
|
||
.equation-card code { display: block; margin: 12px 0; color: #fff; font-size: .82rem; }
|
||
.equation-card p { margin: 0; color: #c0cbd8; font-size: .67rem; }
|
||
|
||
.recipe-selector button { display: grid; gap: 5px; }
|
||
.recipe-selector button span { color: inherit; font-size: .48rem; }
|
||
.recipe-layout {
|
||
display: grid;
|
||
grid-template-columns: minmax(280px, .85fr) 1.15fr;
|
||
gap: 24px;
|
||
margin-top: 24px;
|
||
}
|
||
.recipe-path { display: grid; gap: 1px; border: 1px solid var(--line); background: var(--line); }
|
||
.recipe-path div { display: grid; grid-template-columns: 42px 1fr; align-items: center; min-height: 58px; padding: 10px 14px; background: var(--paper); }
|
||
.recipe-path b { font-size: .7rem; }
|
||
.recipe-contract { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; margin: 0; background: var(--line); border: 1px solid var(--line); }
|
||
.recipe-contract div { padding: 14px; background: var(--paper); }
|
||
.recipe-contract dt { color: var(--muted); font: .5rem var(--mono); }
|
||
.recipe-contract dd { margin: 8px 0 0; font-size: .66rem; line-height: 1.5; }
|
||
.risk-strip { margin-top: 20px; padding: 20px; border: 1px solid #7a6549; background: #efe6d8; }
|
||
.risk-strip b { display: block; margin: 9px 0; color: #6c3c29; }
|
||
.risk-strip p { margin: 0; color: var(--muted); font-size: .66rem; }
|
||
|
||
figcaption {
|
||
padding: 18px 30px;
|
||
border-top: 1px solid var(--line);
|
||
color: var(--muted);
|
||
font-size: .64rem;
|
||
}
|
||
figcaption b { margin-right: 9px; color: var(--copper); font-family: var(--mono); }
|
||
|
||
@media (max-width: 820px) {
|
||
.lab-heading,
|
||
.panel-intro,
|
||
.update-workbench,
|
||
.recipe-layout { grid-template-columns: 1fr; }
|
||
.lab-tabs { grid-template-columns: 1fr 1fr; }
|
||
.lab-tabs button:nth-child(2) { border-right: 0; }
|
||
.sft-sequence { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||
.preference-grid { grid-template-columns: 1fr; }
|
||
.versus { min-height: 110px; }
|
||
.rm-controls { grid-template-columns: 1fr 1fr; }
|
||
.update-flow { display: grid; grid-template-columns: 1fr 1fr; }
|
||
.update-flow > i { display: none; }
|
||
}
|
||
|
||
@media (max-width: 520px) {
|
||
.lab-heading,
|
||
.lab-panel { padding: 20px; }
|
||
.lab-tabs { grid-template-columns: 1fr; }
|
||
.lab-tabs button { border-right: 0; border-bottom: 1px solid var(--line); }
|
||
.sft-controls { align-items: stretch; flex-direction: column; }
|
||
.sft-controls select { width: 100%; margin: 8px 0 0; }
|
||
.sft-sequence,
|
||
.metric-row,
|
||
.rm-controls,
|
||
.recipe-contract,
|
||
.update-flow { grid-template-columns: 1fr; }
|
||
.preference-grid dl { grid-template-columns: 1fr 1fr; }
|
||
}
|
||
</style>
|