---
const sftTokens = [
  { text: "<system>", role: "prompt", probability: 0.94 },
  { text: "请简洁回答", role: "prompt", probability: 0.66 },
  { text: "<user>", role: "prompt", probability: 0.91 },
  { text: "为什么海是蓝色？", role: "prompt", probability: 0.42 },
  { text: "<assistant>", role: "response", probability: 0.87 },
  { text: "阳光", role: "response", probability: 0.72 },
  { text: "进入海水后", role: "response", probability: 0.58 },
  { text: "红光更易被吸收", role: "response", probability: 0.36 },
  { text: "蓝光更易散射返回", role: "response", probability: 0.29 },
  { text: "因此看起来偏蓝", role: "response", probability: 0.63 },
];

const recipes = [
  {
    id: "instructgpt",
    label: "InstructGPT",
    year: "2022",
    family: "Human-feedback RLHF",
    feedback: "人工 demonstration + 人工 K-way ranking",
    regime: "SFT → RM → online PPO-ptx",
    granularity: "Token imitation + response scalar",
    models: "Policy · Reference · RM · Value",
    constraints: "per-token KL + pretraining mix",
    risk: "RM 过优化、标注群体边界、alignment tax",
    path: ["API / 标注者 prompts", "13K SFT demos", "33K ranking prompts → RM", "31K PPO prompts", "PPO-ptx policy"],
  },
  {
    id: "cai",
    label: "Constitutional AI",
    year: "2022",
    family: "Principle-guided RLAIF",
    feedback: "人写原则 + AI critique / revision / preference",
    regime: "SL-CAI → preference model → RL-CAI",
    granularity: "修订示范 + response preference",
    models: "Policy · AI labeler · PM · Value",
    constraints: "constitution + helpfulness objective",
    risk: "原则覆盖、teacher 偏差、无害但回避",
    path: ["Constitution", "Self-critique", "Revised SFT data", "AI preferences → PM", "RL-CAI"],
  },
  {
    id: "dpo",
    label: "DPO",
    year: "2023",
    family: "Offline direct preference",
    feedback: "固定 chosen / rejected pairs",
    regime: "SFT reference → offline DPO",
    granularity: "sequence log-ratio pair",
    models: "Policy · frozen Reference",
    constraints: "β × policy/reference ratio",
    risk: "pair 分布陈旧、长度口径、reference 选择",
    path: ["Prompt", "Sample pairs", "Preference label", "Reference log-prob", "Direct policy update"],
  },
  {
    id: "deepseekv2",
    label: "DeepSeek-V2",
    year: "2024",
    family: "Multi-reward GRPO",
    feedback: "人工偏好 + RM + 编译器/答案规则",
    regime: "SFT → reasoning RL → preference RL",
    granularity: "group-relative outcome reward",
    models: "Policy · Reference · multiple RMs",
    constraints: "KL + rule / helpful / safety rewards",
    risk: "alignment tax、online/offline 分布、RM 可靠性",
    path: ["SFT", "Reasoning RM", "GRPO stage 1", "Helpful / Safety / Rule", "GRPO stage 2"],
  },
  {
    id: "deepseekv4",
    label: "DeepSeek-V4",
    year: "2026",
    family: "SFT + GRPO + GRM",
    feedback: "规则 verifier + rubric-guided generative reward",
    regime: "SFT → unified domain GRPO",
    granularity: "outcome + generated rubric score",
    models: "Policy · Reference · Verifier / GRM",
    constraints: "规则优先、rubric、QAT deployment constraint",
    risk: "GRM 可被利用、rubric 缺口、环境覆盖不足",
    path: ["Curated SFT", "Rule-verifiable tasks", "Hard tasks → GRM", "GRPO", "Domain-aligned model"],
  },
  {
    id: "k3",
    label: "Kimi K3",
    year: "2026",
    family: "Multi-effort expert integration",
    feedback: "规则/环境/Agentic GRM + 9 teachers",
    regime: "SFT → 9 RL experts → MOPD",
    granularity: "outcome reward + per-token teacher signal",
    models: "Student · 9 Teachers · Verifiers / GRM",
    constraints: "partial-rollout clipping + budget + verbosity",
    risk: "长轨迹陈旧、teacher 偏差、reward hacking",
    path: ["Cold-start SFT", "3 domains × 3 efforts", "Partial-rollout RL", "Student on-policy rollout", "MOPD consolidation"],
  },
];
---

<figure
  class="alignment-lab"
  data-alignment-lab
  data-sft-tokens={JSON.stringify(sftTokens)}
  data-recipes={JSON.stringify(recipes)}
>
  <div class="lab-heading">
    <div>
      <span class="micro-label">INTERACTIVE / FOUR-VIEW LAB</span>
      <h3>把“对齐”拆成四个不相互冒充的实验</h3>
    </div>
    <p>所有数字都是教学模拟，用来展示目标函数和反馈结构；不是任何模型的训练日志或产品指标。</p>
  </div>

  <div class="lab-tabs" role="tablist" aria-label="后训练四联实验">
    <button type="button" role="tab" aria-selected="true" aria-controls="alignment-panel-sft" id="alignment-tab-sft" data-alignment-tab="sft">
      <span>01</span><b>SFT 显微镜</b><small>哪些 Token 产生梯度</small>
    </button>
    <button type="button" role="tab" aria-selected="false" aria-controls="alignment-panel-preference" id="alignment-tab-preference" data-alignment-tab="preference" tabindex="-1">
      <span>02</span><b>偏好与 RM</b><small>比较怎样变成标量</small>
    </button>
    <button type="button" role="tab" aria-selected="false" aria-controls="alignment-panel-update" id="alignment-tab-update" data-alignment-tab="update" tabindex="-1">
      <span>03</span><b>PPO / DPO</b><small>两种更新合同</small>
    </button>
    <button type="button" role="tab" aria-selected="false" aria-controls="alignment-panel-recipe" id="alignment-tab-recipe" data-alignment-tab="recipe" tabindex="-1">
      <span>04</span><b>现代配方</b><small>2022 → K3</small>
    </button>
  </div>

  <section class="lab-panel" role="tabpanel" id="alignment-panel-sft" aria-labelledby="alignment-tab-sft" data-alignment-panel="sft">
    <div class="panel-intro">
      <div><span>QUESTION</span><h4>SFT 到底在模仿 prompt，还是只模仿 assistant？</h4></div>
      <p>真实 chat template 会把角色标记也编码成 Token；是否计入 loss 由 mask 决定，而不是由颜色决定。</p>
    </div>

    <div class="sft-controls">
      <div class="mode-switch" role="group" aria-label="选择 SFT loss mask">
        <button type="button" class="active" data-sft-mask="response">只训练 Assistant</button>
        <button type="button" data-sft-mask="all">全序列训练</button>
      </div>
      <label>
        Demonstration 质量
        <select data-sft-quality>
          <option value="expert" selected>专家、简洁、事实正确</option>
          <option value="verbose">基本正确，但冗长</option>
          <option value="unsafe">格式流畅，但事实错误</option>
        </select>
      </label>
    </div>

    <div class="sft-sequence" data-sft-sequence>
      {sftTokens.map((token, index) => (
        <div data-sft-token={index} data-role={token.role}>
          <span>{token.role === "prompt" ? "CONDITION" : "TARGET"}</span>
          <b>{token.text}</b>
          <i><em data-sft-prob={index}>{Math.round(token.probability * 100)}%</em><small data-sft-loss={index}>—</small></i>
        </div>
      ))}
    </div>

    <div class="metric-row">
      <article><span>LOSS TOKENS</span><b data-sft-active>6 / 10</b><p>真正进入交叉熵求和的 Token</p></article>
      <article><span>MEAN NLL</span><b data-sft-nll>0.72</b><p>只反映这条示范的拟合难度</p></article>
      <article><span>TEACHER FORCING</span><b>ON</b><p>第 t 步总能看到参考答案的前缀</p></article>
    </div>

    <div class="lab-takeaway">
      <b data-sft-reading>SFT 正在学习一条高质量示范，但它没有直接比较其他可能回答。</b>
      <p>如果 demonstration 本身错误，交叉熵仍会忠实提高错误 Token 的概率；loss 并不知道“事实正确”。</p>
    </div>
  </section>

  <section class="lab-panel" role="tabpanel" id="alignment-panel-preference" aria-labelledby="alignment-tab-preference" data-alignment-panel="preference" hidden>
    <div class="panel-intro">
      <div><span>QUESTION</span><h4>为什么“chosen”只是相对更好，不一定是好答案？</h4></div>
      <p>滑块控制的是一个教学 reward model 的偏好权重；真实 RM 会从标注数据中学习这些权重与 shortcut。</p>
    </div>

    <div class="preference-presets" role="group" aria-label="选择偏好场景">
      <button type="button" class="active" data-pref-preset="evidence">事实胜过语气</button>
      <button type="button" data-pref-preset="length">长度 Shortcut</button>
      <button type="button" data-pref-preset="bothbad">两个都不好</button>
      <button type="button" data-pref-preset="disagree">标注者分歧</button>
    </div>

    <div class="preference-grid">
      <article data-answer-card="a">
        <span>RESPONSE A</span>
        <h4 data-answer-title="a">短而有证据</h4>
        <p data-answer-copy="a">指出海水对长波吸收更强，并说明散射与反射共同影响观察颜色。</p>
        <dl>
          <div><dt>事实</dt><dd data-feature="a-fact">92</dd></div>
          <div><dt>帮助</dt><dd data-feature="a-help">78</dd></div>
          <div><dt>风格</dt><dd data-feature="a-style">58</dd></div>
          <div><dt>长度</dt><dd data-feature="a-length">34</dd></div>
        </dl>
        <strong data-reward="a">+1.42</strong>
      </article>
      <div class="versus">
        <span>BRADLEY–TERRY</span>
        <b data-pref-probability>74%</b>
        <small data-pref-choice>P(A ≻ B)</small>
        <i><em data-pref-meter></em></i>
      </div>
      <article data-answer-card="b">
        <span>RESPONSE B</span>
        <h4 data-answer-title="b">流畅但错误</h4>
        <p data-answer-copy="b">写得更长、更有自信，却把海水颜色完全归因于天空倒影。</p>
        <dl>
          <div><dt>事实</dt><dd data-feature="b-fact">24</dd></div>
          <div><dt>帮助</dt><dd data-feature="b-help">46</dd></div>
          <div><dt>风格</dt><dd data-feature="b-style">86</dd></div>
          <div><dt>长度</dt><dd data-feature="b-length">78</dd></div>
        </dl>
        <strong data-reward="b">+0.39</strong>
      </article>
    </div>

    <div class="rm-controls">
      <label>事实权重<input type="range" min="0" max="100" value="70" data-rm-weight="fact" /><output data-rm-label="fact">0.70</output></label>
      <label>帮助权重<input type="range" min="0" max="100" value="50" data-rm-weight="help" /><output data-rm-label="help">0.50</output></label>
      <label>风格权重<input type="range" min="0" max="100" value="25" data-rm-weight="style" /><output data-rm-label="style">0.25</output></label>
      <label>长度权重<input type="range" min="-50" max="80" value="5" data-rm-weight="length" /><output data-rm-label="length">0.05</output></label>
      <label>标签噪声<input type="range" min="0" max="45" value="8" data-rm-noise /><output data-rm-noise-label>8%</output></label>
    </div>

    <div class="lab-takeaway">
      <b data-pref-reading>这个 RM 更看重事实，因此短回答 A 获得更高偏好概率。</b>
      <p>概率来自 reward difference，不是绝对质量证书；如果两个回答都很差，模型仍会被要求选出一个相对赢家。</p>
    </div>
  </section>

  <section class="lab-panel" role="tabpanel" id="alignment-panel-update" aria-labelledby="alignment-tab-update" data-alignment-panel="update" hidden>
    <div class="panel-intro">
      <div><span>QUESTION</span><h4>PPO 与 DPO 到底在哪一步分叉？</h4></div>
      <p>二者都可从 pairwise preference 出发，但 PPO 显式拟合 RM 并向当前 policy 采样；经典 DPO 直接消费固定 pairs。</p>
    </div>

    <div class="update-switch" role="group" aria-label="选择偏好优化方法">
      <button type="button" class="active" data-update-method="ppo">PPO / explicit RM</button>
      <button type="button" data-update-method="dpo">DPO / implicit RM</button>
    </div>

    <div class="update-workbench">
      <div class="update-flow" data-update-flow>
        <div><span>01</span><b data-update-step="0">Current policy rollout</b></div>
        <i>→</i>
        <div><span>02</span><b data-update-step="1">Reward model score</b></div>
        <i>→</i>
        <div><span>03</span><b data-update-step="2">Advantage + critic</b></div>
        <i>→</i>
        <div><span>04</span><b data-update-step="3">Clipped policy update</b></div>
      </div>

      <div class="update-controls">
        <label>
          Reward / preference gap
          <input type="range" min="-100" max="200" value="80" data-update-gap />
          <output data-update-gap-label>+0.80</output>
        </label>
        <label>
          KL / β 约束
          <input type="range" min="1" max="100" value="25" data-update-beta />
          <output data-update-beta-label>0.25</output>
        </label>
        <label>
          Policy 相对 reference 位移
          <input type="range" min="-100" max="160" value="40" data-update-shift />
          <output data-update-shift-label>+0.40</output>
        </label>
        <label>
          Pair / rollout 陈旧度
          <input type="range" min="0" max="100" value="15" data-update-stale />
          <output data-update-stale-label>15%</output>
        </label>
      </div>
    </div>

    <div class="metric-row">
      <article><span>TRAINING SIGNAL</span><b data-update-signal>+0.41</b><p data-update-signal-note>RM advantage 经 PPO clip 与 KL 后的教学量</p></article>
      <article><span>LIVE MODELS</span><b data-update-models>4</b><p data-update-model-note>policy · reference · RM · value</p></article>
      <article><span>DATA REGIME</span><b data-update-regime>ONLINE</b><p data-update-regime-note>需要接近当前 policy 的 rollout</p></article>
    </div>

    <div class="equation-card">
      <span data-update-kicker>PPO SKETCH</span>
      <code data-update-equation>min(ρA, clip(ρ)A) − β KL(π || πref)</code>
      <p data-update-reading>当 policy 已明显偏离 behavior policy，importance ratio 与 critic 误差会让更新更脆弱。</p>
    </div>
  </section>

  <section class="lab-panel" role="tabpanel" id="alignment-panel-recipe" aria-labelledby="alignment-tab-recipe" data-alignment-panel="recipe" hidden>
    <div class="panel-intro">
      <div><span>QUESTION</span><h4>现代 post-training 为什么不再是一条固定的 SFT→RM→PPO 流水线？</h4></div>
      <p>用同一组字段比较反馈来源、训练制度、监督粒度、约束与失效模式，避免只比较算法缩写。</p>
    </div>

    <div class="recipe-selector" role="group" aria-label="选择 post-training 配方">
      {recipes.map((recipe, index) => (
        <button type="button" class:list={{ active: index === 0 }} data-recipe-id={recipe.id}>
          <span>{recipe.year}</span>{recipe.label}
        </button>
      ))}
    </div>

    <div class="recipe-layout">
      <div class="recipe-path" data-recipe-path>
        {recipes[0].path.map((step, index) => (
          <div>
            <span>{String(index + 1).padStart(2, "0")}</span>
            <b data-recipe-step>{step}</b>
          </div>
        ))}
      </div>
      <dl class="recipe-contract">
        <div><dt>FAMILY</dt><dd data-recipe-family>{recipes[0].family}</dd></div>
        <div><dt>FEEDBACK</dt><dd data-recipe-feedback>{recipes[0].feedback}</dd></div>
        <div><dt>REGIME</dt><dd data-recipe-regime>{recipes[0].regime}</dd></div>
        <div><dt>GRANULARITY</dt><dd data-recipe-granularity>{recipes[0].granularity}</dd></div>
        <div><dt>LIVE MODELS</dt><dd data-recipe-models>{recipes[0].models}</dd></div>
        <div><dt>CONSTRAINTS</dt><dd data-recipe-constraints>{recipes[0].constraints}</dd></div>
      </dl>
    </div>

    <div class="risk-strip">
      <span>PRIMARY FAILURE MODE</span>
      <b data-recipe-risk>{recipes[0].risk}</b>
      <p>这是一张角色合同，不是排行榜；不同配方面对的任务、数据与预算并不相同。</p>
    </div>
  </section>

  <figcaption>
    <b>交互图 10</b>
    四个实验依次回答：示范怎样产生 Token 梯度、偏好怎样变成代理标量、policy 怎样更新，以及现代配方怎样组合多种反馈。
  </figcaption>
</figure>

<script>
  type SftToken = { text: string; role: "prompt" | "response"; probability: number };
  type Recipe = {
    id: string;
    label: string;
    year: string;
    family: string;
    feedback: string;
    regime: string;
    granularity: string;
    models: string;
    constraints: string;
    risk: string;
    path: string[];
  };

  document.querySelectorAll<HTMLElement>("[data-alignment-lab]").forEach((root) => {
    const tabs = [...root.querySelectorAll<HTMLButtonElement>("[data-alignment-tab]")];
    const panels = [...root.querySelectorAll<HTMLElement>("[data-alignment-panel]")];
    const sftTokens = JSON.parse(root.dataset.sftTokens ?? "[]") as SftToken[];
    const recipes = JSON.parse(root.dataset.recipes ?? "[]") as Recipe[];

    const setText = (selector: string, text: string) => {
      const node = root.querySelector<HTMLElement>(selector);
      if (node) node.textContent = text;
    };
    const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value));
    const sigmoid = (value: number) => 1 / (1 + Math.exp(-value));

    const activateTab = (name: string, focus = false) => {
      tabs.forEach((tab) => {
        const active = tab.dataset.alignmentTab === name;
        tab.setAttribute("aria-selected", String(active));
        tab.tabIndex = active ? 0 : -1;
        if (active && focus) tab.focus();
      });
      panels.forEach((panel) => {
        panel.hidden = panel.dataset.alignmentPanel !== name;
      });
    };
    tabs.forEach((tab, index) => {
      tab.addEventListener("click", () => activateTab(tab.dataset.alignmentTab ?? "sft"));
      tab.addEventListener("keydown", (event) => {
        let next = index;
        if (event.key === "ArrowRight") next = (index + 1) % tabs.length;
        else if (event.key === "ArrowLeft") next = (index - 1 + tabs.length) % tabs.length;
        else if (event.key === "Home") next = 0;
        else if (event.key === "End") next = tabs.length - 1;
        else return;
        event.preventDefault();
        activateTab(tabs[next].dataset.alignmentTab ?? "sft", true);
      });
    });

    let sftMask = "response";
    const qualitySelect = root.querySelector<HTMLSelectElement>("[data-sft-quality]");
    const qualityMultipliers: Record<string, { probabilities: number[]; reading: string }> = {
      expert: {
        probabilities: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        reading: "SFT 正在学习一条高质量示范，但它没有直接比较其他可能回答。",
      },
      verbose: {
        probabilities: [1, 1, 1, 1, 1, 0.92, 0.84, 0.78, 0.72, 0.68],
        reading: "冗长示范提供更多稠密梯度，也会把重复、铺垫和表达习惯一起写进 policy。",
      },
      unsafe: {
        probabilities: [1, 1, 1, 1, 1, 0.7, 0.46, 0.3, 0.24, 0.34],
        reading: "交叉熵不会识别事实错误：只要它是 target，错误回答同样会得到正向模仿梯度。",
      },
    };
    const renderSft = () => {
      const quality = qualitySelect?.value ?? "expert";
      const profile = qualityMultipliers[quality];
      let active = 0;
      let totalNll = 0;
      sftTokens.forEach((token, index) => {
        const contributes = sftMask === "all" || token.role === "response";
        const probability = clamp(token.probability * (profile.probabilities[index] ?? 1), 0.04, 0.99);
        const loss = -Math.log(probability);
        const card = root.querySelector<HTMLElement>(`[data-sft-token="${index}"]`);
        card?.toggleAttribute("data-active", contributes);
        setText(`[data-sft-prob="${index}"]`, `${Math.round(probability * 100)}%`);
        setText(`[data-sft-loss="${index}"]`, contributes ? `NLL ${loss.toFixed(2)}` : "MASKED");
        if (contributes) {
          active += 1;
          totalNll += loss;
        }
      });
      setText("[data-sft-active]", `${active} / ${sftTokens.length}`);
      setText("[data-sft-nll]", (totalNll / Math.max(active, 1)).toFixed(2));
      setText("[data-sft-reading]", profile.reading);
      root.querySelectorAll<HTMLButtonElement>("[data-sft-mask]").forEach((button) => {
        button.classList.toggle("active", button.dataset.sftMask === sftMask);
      });
    };
    root.querySelectorAll<HTMLButtonElement>("[data-sft-mask]").forEach((button) => {
      button.addEventListener("click", () => {
        sftMask = button.dataset.sftMask ?? "response";
        renderSft();
      });
    });
    qualitySelect?.addEventListener("change", renderSft);
    renderSft();

    type Answer = { title: string; copy: string; fact: number; help: number; style: number; length: number };
    const preferenceProfiles: Record<string, { a: Answer; b: Answer; weights: number[]; noise: number; reading: string }> = {
      evidence: {
        a: { title: "短而有证据", copy: "指出海水对长波吸收更强，并说明散射与反射共同影响观察颜色。", fact: 92, help: 78, style: 58, length: 34 },
        b: { title: "流畅但错误", copy: "写得更长、更有自信，却把海水颜色完全归因于天空倒影。", fact: 24, help: 46, style: 86, length: 78 },
        weights: [70, 50, 25, 5], noise: 8,
        reading: "这个 RM 更看重事实，因此短回答 A 获得更高偏好概率。",
      },
      length: {
        a: { title: "短而正确", copy: "答案正确、简洁，但没有铺陈背景或重复结论。", fact: 90, help: 72, style: 52, length: 24 },
        b: { title: "长而空泛", copy: "分成很多段并反复总结，信息增量有限，但表面上显得完整。", fact: 54, help: 58, style: 78, length: 96 },
        weights: [35, 35, 35, 65], noise: 10,
        reading: "长度被赋予过高权重时，RM 会把“写得多”误当成“帮助更大”。",
      },
      bothbad: {
        a: { title: "简短猜测", copy: "给出一个没有证据的猜测，遗漏关键机制。", fact: 28, help: 24, style: 42, length: 20 },
        b: { title: "自信编造", copy: "用流畅长文包装错误因果，还加入不存在的数字。", fact: 12, help: 30, style: 76, length: 82 },
        weights: [60, 45, 30, 5], noise: 12,
        reading: "A 只是“两坏相较取其轻”；chosen 标签不能把它升级成高质量 demonstration。",
      },
      disagree: {
        a: { title: "直接纠错", copy: "明确指出用户前提错误，事实准确但语气生硬。", fact: 94, help: 62, style: 28, length: 38 },
        b: { title: "温和迎合", copy: "先认同用户，再给模糊建议；更悦耳但保留错误前提。", fact: 42, help: 66, style: 91, length: 58 },
        weights: [48, 50, 55, 8], noise: 38,
        reading: "当标注者在“直接事实”与“关系体验”之间分歧，单一标量会把差异压成平均偏好。",
      },
    };
    let currentPreference = preferenceProfiles.evidence;
    const weightInputs = [...root.querySelectorAll<HTMLInputElement>("[data-rm-weight]")];
    const noiseInput = root.querySelector<HTMLInputElement>("[data-rm-noise]");
    const updateAnswer = (side: "a" | "b", answer: Answer) => {
      setText(`[data-answer-title="${side}"]`, answer.title);
      setText(`[data-answer-copy="${side}"]`, answer.copy);
      (["fact", "help", "style", "length"] as const).forEach((feature) => {
        setText(`[data-feature="${side}-${feature}"]`, String(answer[feature]));
      });
    };
    const renderPreference = () => {
      const weights = Object.fromEntries(weightInputs.map((input) => [input.dataset.rmWeight ?? "", Number(input.value) / 100])) as Record<string, number>;
      const noise = Number(noiseInput?.value ?? 0) / 100;
      const score = (answer: Answer) => (
        answer.fact * weights.fact +
        answer.help * weights.help +
        answer.style * weights.style +
        answer.length * weights.length
      ) / 100 - 0.95;
      const rewardA = score(currentPreference.a);
      const rewardB = score(currentPreference.b);
      const cleanProbability = sigmoid(rewardA - rewardB);
      const probability = cleanProbability * (1 - noise) + 0.5 * noise;
      setText('[data-reward="a"]', `${rewardA >= 0 ? "+" : ""}${rewardA.toFixed(2)}`);
      setText('[data-reward="b"]', `${rewardB >= 0 ? "+" : ""}${rewardB.toFixed(2)}`);
      setText("[data-pref-probability]", `${Math.round(probability * 100)}%`);
      setText("[data-pref-choice]", probability >= 0.5 ? "P(A ≻ B)" : "P(B ≻ A)");
      const meter = root.querySelector<HTMLElement>("[data-pref-meter]");
      if (meter) meter.style.width = `${probability * 100}%`;
      weightInputs.forEach((input) => {
        const key = input.dataset.rmWeight ?? "";
        setText(`[data-rm-label="${key}"]`, (Number(input.value) / 100).toFixed(2));
      });
      setText("[data-rm-noise-label]", `${Math.round(noise * 100)}%`);
      setText("[data-pref-reading]", currentPreference.reading);
    };
    root.querySelectorAll<HTMLButtonElement>("[data-pref-preset]").forEach((button) => {
      button.addEventListener("click", () => {
        currentPreference = preferenceProfiles[button.dataset.prefPreset ?? "evidence"];
        updateAnswer("a", currentPreference.a);
        updateAnswer("b", currentPreference.b);
        weightInputs.forEach((input, index) => { input.value = String(currentPreference.weights[index]); });
        if (noiseInput) noiseInput.value = String(currentPreference.noise);
        root.querySelectorAll<HTMLButtonElement>("[data-pref-preset]").forEach((item) => item.classList.toggle("active", item === button));
        renderPreference();
      });
    });
    [...weightInputs, noiseInput].forEach((input) => input?.addEventListener("input", renderPreference));
    renderPreference();

    let updateMethod = "ppo";
    const gapInput = root.querySelector<HTMLInputElement>("[data-update-gap]");
    const betaInput = root.querySelector<HTMLInputElement>("[data-update-beta]");
    const shiftInput = root.querySelector<HTMLInputElement>("[data-update-shift]");
    const staleInput = root.querySelector<HTMLInputElement>("[data-update-stale]");
    const renderUpdate = () => {
      const gap = Number(gapInput?.value ?? 80) / 100;
      const beta = Number(betaInput?.value ?? 25) / 100;
      const shift = Number(shiftInput?.value ?? 40) / 100;
      const stale = Number(staleInput?.value ?? 15) / 100;
      setText("[data-update-gap-label]", `${gap >= 0 ? "+" : ""}${gap.toFixed(2)}`);
      setText("[data-update-beta-label]", beta.toFixed(2));
      setText("[data-update-shift-label]", `${shift >= 0 ? "+" : ""}${shift.toFixed(2)}`);
      setText("[data-update-stale-label]", `${Math.round(stale * 100)}%`);
      root.querySelectorAll<HTMLButtonElement>("[data-update-method]").forEach((button) => {
        button.classList.toggle("active", button.dataset.updateMethod === updateMethod);
      });
      if (updateMethod === "ppo") {
        const ratio = Math.exp(shift * (1 + stale));
        const clipped = clamp(ratio, 0.8, 1.2);
        const signal = Math.min(ratio * gap, clipped * gap) - beta * Math.abs(shift);
        ["Current policy rollout", "Reward model score", "Advantage + critic", "Clipped policy update"].forEach((step, index) => setText(`[data-update-step="${index}"]`, step));
        setText("[data-update-signal]", `${signal >= 0 ? "+" : ""}${signal.toFixed(2)}`);
        setText("[data-update-signal-note]", "RM advantage 经 PPO clip 与 KL 后的教学量");
        setText("[data-update-models]", "4");
        setText("[data-update-model-note]", "policy · reference · RM · value");
        setText("[data-update-regime]", "ONLINE");
        setText("[data-update-regime-note]", "需要接近当前 policy 的 rollout");
        setText("[data-update-kicker]", "PPO SKETCH");
        setText("[data-update-equation]", "min(ρA, clip(ρ)A) − β KL(π || πref)");
        setText("[data-update-reading]", stale > 0.55
          ? "rollout 已明显陈旧；importance ratio、critic 与长轨迹 credit 会让更新更脆弱。"
          : "PPO 显式消费 rollout 与 scalar reward；clip 约束单次 policy 位移，但不修复错误奖励。");
      } else {
        const implicitMargin = beta * (gap - shift * stale);
        const loss = -Math.log(sigmoid(implicitMargin));
        ["Fixed preference pairs", "Policy / reference log-probs", "Implicit reward margin", "Classification-style update"].forEach((step, index) => setText(`[data-update-step="${index}"]`, step));
        setText("[data-update-signal]", implicitMargin.toFixed(2));
        setText("[data-update-signal-note]", "β × chosen/rejected 的相对 log-ratio margin");
        setText("[data-update-models]", "2");
        setText("[data-update-model-note]", "trainable policy · frozen reference");
        setText("[data-update-regime]", "OFFLINE");
        setText("[data-update-regime-note]", "经典 DPO 不在训练中向当前 policy 采样");
        setText("[data-update-kicker]", "DPO SKETCH");
        setText("[data-update-equation]", `−log σ(β Δlog(π/πref)) = ${loss.toFixed(2)}`);
        setText("[data-update-reading]", stale > 0.55
          ? "pair 与当前 policy 差距很大：目标仍能计算，但它回答的是旧候选分布上的偏好。"
          : "DPO 省去显式 RM 与在线 RL loop；偏好 pair、reference、β 和分布匹配仍然存在。");
      }
    };
    root.querySelectorAll<HTMLButtonElement>("[data-update-method]").forEach((button) => {
      button.addEventListener("click", () => {
        updateMethod = button.dataset.updateMethod ?? "ppo";
        renderUpdate();
      });
    });
    [gapInput, betaInput, shiftInput, staleInput].forEach((input) => input?.addEventListener("input", renderUpdate));
    renderUpdate();

    const recipeButtons = [...root.querySelectorAll<HTMLButtonElement>("[data-recipe-id]")];
    const renderRecipe = (id: string) => {
      const recipe = recipes.find((item) => item.id === id) ?? recipes[0];
      recipeButtons.forEach((button) => button.classList.toggle("active", button.dataset.recipeId === recipe.id));
      const stepNodes = [...root.querySelectorAll<HTMLElement>("[data-recipe-step]")];
      stepNodes.forEach((node, index) => { node.textContent = recipe.path[index] ?? ""; });
      (["family", "feedback", "regime", "granularity", "models", "constraints", "risk"] as const).forEach((field) => {
        setText(`[data-recipe-${field}]`, recipe[field]);
      });
    };
    recipeButtons.forEach((button) => button.addEventListener("click", () => renderRecipe(button.dataset.recipeId ?? "instructgpt")));
    renderRecipe("instructgpt");
  });
</script>

<style>
  .alignment-lab {
    max-width: 1020px;
    margin: 42px 0;
    border: 1px solid var(--line);
    background: var(--paper-raised);
  }

  .lab-heading,
  .panel-intro {
    display: grid;
    grid-template-columns: 1fr minmax(250px, 350px);
    gap: 34px;
    align-items: end;
  }

  .lab-heading {
    padding: 30px;
    border-bottom: 1px solid var(--line);
  }

  .lab-heading h3 {
    margin-top: 10px;
    font-size: 1.55rem;
  }

  .lab-heading p,
  .panel-intro p {
    color: var(--muted);
    font-size: .72rem;
    line-height: 1.75;
  }

  .lab-tabs {
    display: grid;
    grid-template-columns: repeat(4, minmax(0, 1fr));
    border-bottom: 1px solid var(--line);
  }

  .lab-tabs button {
    display: grid;
    gap: 7px;
    min-height: 92px;
    padding: 15px 17px;
    border: 0;
    border-right: 1px solid var(--line);
    background: transparent;
    color: var(--muted);
    text-align: left;
  }

  .lab-tabs button:last-child { border-right: 0; }
  .lab-tabs button[aria-selected="true"] { color: white; background: var(--navy); }
  .lab-tabs span,
  .panel-intro span,
  .preference-grid > article > span,
  .versus span,
  .risk-strip span,
  .equation-card span {
    color: var(--copper);
    font: .58rem/1.2 var(--mono);
    letter-spacing: .09em;
  }
  .lab-tabs b { font: 600 .85rem/1.15 var(--sans); }
  .lab-tabs small { font: .54rem/1.2 var(--mono); }

  .lab-panel { padding: 30px; }
  .panel-intro { margin-bottom: 28px; }
  .panel-intro h4 { margin-top: 8px; font-size: 1.28rem; }

  .mode-switch,
  .preference-presets,
  .update-switch,
  .recipe-selector {
    display: flex;
    flex-wrap: wrap;
    gap: 8px;
  }

  .mode-switch button,
  .preference-presets button,
  .update-switch button,
  .recipe-selector button {
    padding: 10px 13px;
    border: 1px solid var(--line);
    background: var(--paper);
    color: var(--muted);
    font: .62rem var(--mono);
  }

  .mode-switch button.active,
  .preference-presets button.active,
  .update-switch button.active,
  .recipe-selector button.active {
    border-color: var(--navy);
    background: var(--navy);
    color: white;
  }

  .sft-controls {
    display: flex;
    justify-content: space-between;
    gap: 20px;
    align-items: center;
  }

  .sft-controls label,
  .update-controls label,
  .rm-controls label {
    color: var(--muted);
    font: .62rem var(--mono);
  }

  select {
    min-height: 38px;
    margin-left: 8px;
    border: 1px solid var(--line);
    background: var(--paper);
    color: var(--ink);
  }

  .sft-sequence {
    display: grid;
    grid-template-columns: repeat(5, minmax(0, 1fr));
    gap: 1px;
    margin-top: 24px;
    border: 1px solid var(--line);
    background: var(--line);
  }

  .sft-sequence > div {
    min-height: 126px;
    padding: 13px;
    background: var(--paper);
    opacity: .56;
  }
  .sft-sequence > div[data-active] { opacity: 1; box-shadow: inset 0 -3px var(--copper); }
  .sft-sequence > div[data-role="response"] { background: #f0eadf; }
  .sft-sequence span { color: var(--muted); font: .5rem var(--mono); }
  .sft-sequence b { display: block; min-height: 42px; margin: 12px 0; font-size: .7rem; }
  .sft-sequence i { display: flex; justify-content: space-between; gap: 6px; font-style: normal; }
  .sft-sequence em { color: var(--navy); font: normal 600 .62rem var(--mono); }
  .sft-sequence small { color: var(--copper); font: .52rem var(--mono); }

  .metric-row {
    display: grid;
    grid-template-columns: repeat(3, 1fr);
    gap: 1px;
    margin-top: 24px;
    border: 1px solid var(--line);
    background: var(--line);
  }
  .metric-row article { padding: 18px; background: var(--paper); }
  .metric-row span { color: var(--muted); font: .52rem var(--mono); }
  .metric-row b { display: block; margin: 11px 0 6px; color: var(--navy); font: 600 1.25rem var(--serif); }
  .metric-row p { margin: 0; color: var(--muted); font-size: .62rem; }

  .lab-takeaway {
    margin-top: 20px;
    padding: 18px 20px;
    border-left: 3px solid var(--copper);
    background: #f1eadf;
  }
  .lab-takeaway b { color: var(--navy); }
  .lab-takeaway p { margin: 8px 0 0; color: var(--muted); font-size: .7rem; }

  .preference-grid {
    display: grid;
    grid-template-columns: 1fr 120px 1fr;
    gap: 1px;
    margin-top: 22px;
    border: 1px solid var(--line);
    background: var(--line);
  }
  .preference-grid > article { padding: 20px; background: var(--paper); }
  .preference-grid h4 { margin: 10px 0; font-size: 1rem; }
  .preference-grid p { min-height: 64px; color: var(--muted); font-size: .68rem; }
  .preference-grid dl { display: grid; grid-template-columns: repeat(4, 1fr); gap: 1px; margin: 18px 0; background: var(--line); }
  .preference-grid dl div { padding: 9px 5px; background: white; text-align: center; }
  .preference-grid dt { color: var(--muted); font: .48rem var(--mono); }
  .preference-grid dd { margin: 6px 0 0; font: 600 .72rem var(--mono); }
  .preference-grid article > strong { color: var(--navy); font: 600 1.35rem var(--serif); }

  .versus {
    display: flex;
    flex-direction: column;
    align-items: center;
    justify-content: center;
    gap: 10px;
    padding: 12px;
    background: var(--navy);
    color: white;
    text-align: center;
  }
  .versus b { font: 600 1.6rem var(--serif); }
  .versus small { font: .5rem var(--mono); }
  .versus i { width: 100%; height: 5px; background: #526477; }
  .versus em { display: block; height: 100%; background: var(--copper); }

  .rm-controls,
  .update-controls {
    display: grid;
    grid-template-columns: repeat(5, minmax(0, 1fr));
    gap: 14px;
    margin-top: 20px;
    padding: 18px;
    border: 1px solid var(--line);
    background: #f7f3eb;
  }
  .rm-controls label,
  .update-controls label { display: grid; gap: 8px; }
  input[type="range"] { width: 100%; accent-color: var(--copper); }
  output { color: var(--navy); font-weight: 700; }

  .update-workbench {
    display: grid;
    grid-template-columns: 1fr 300px;
    gap: 22px;
    margin-top: 22px;
  }
  .update-flow {
    display: flex;
    align-items: stretch;
    gap: 8px;
  }
  .update-flow div {
    display: grid;
    flex: 1;
    align-content: center;
    gap: 8px;
    min-height: 124px;
    padding: 14px;
    border: 1px solid var(--line);
    background: var(--paper);
  }
  .update-flow span,
  .recipe-path span { color: var(--copper); font: .55rem var(--mono); }
  .update-flow b { font-size: .67rem; }
  .update-flow i { align-self: center; color: var(--copper); font-style: normal; }
  .update-controls { grid-template-columns: 1fr; margin: 0; }

  .equation-card {
    margin-top: 20px;
    padding: 20px;
    background: #25364b;
    color: white;
  }
  .equation-card code { display: block; margin: 12px 0; color: #fff; font-size: .82rem; }
  .equation-card p { margin: 0; color: #c0cbd8; font-size: .67rem; }

  .recipe-selector button { display: grid; gap: 5px; }
  .recipe-selector button span { color: inherit; font-size: .48rem; }
  .recipe-layout {
    display: grid;
    grid-template-columns: minmax(280px, .85fr) 1.15fr;
    gap: 24px;
    margin-top: 24px;
  }
  .recipe-path { display: grid; gap: 1px; border: 1px solid var(--line); background: var(--line); }
  .recipe-path div { display: grid; grid-template-columns: 42px 1fr; align-items: center; min-height: 58px; padding: 10px 14px; background: var(--paper); }
  .recipe-path b { font-size: .7rem; }
  .recipe-contract { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; margin: 0; background: var(--line); border: 1px solid var(--line); }
  .recipe-contract div { padding: 14px; background: var(--paper); }
  .recipe-contract dt { color: var(--muted); font: .5rem var(--mono); }
  .recipe-contract dd { margin: 8px 0 0; font-size: .66rem; line-height: 1.5; }
  .risk-strip { margin-top: 20px; padding: 20px; border: 1px solid #7a6549; background: #efe6d8; }
  .risk-strip b { display: block; margin: 9px 0; color: #6c3c29; }
  .risk-strip p { margin: 0; color: var(--muted); font-size: .66rem; }

  figcaption {
    padding: 18px 30px;
    border-top: 1px solid var(--line);
    color: var(--muted);
    font-size: .64rem;
  }
  figcaption b { margin-right: 9px; color: var(--copper); font-family: var(--mono); }

  @media (max-width: 820px) {
    .lab-heading,
    .panel-intro,
    .update-workbench,
    .recipe-layout { grid-template-columns: 1fr; }
    .lab-tabs { grid-template-columns: 1fr 1fr; }
    .lab-tabs button:nth-child(2) { border-right: 0; }
    .sft-sequence { grid-template-columns: repeat(2, minmax(0, 1fr)); }
    .preference-grid { grid-template-columns: 1fr; }
    .versus { min-height: 110px; }
    .rm-controls { grid-template-columns: 1fr 1fr; }
    .update-flow { display: grid; grid-template-columns: 1fr 1fr; }
    .update-flow > i { display: none; }
  }

  @media (max-width: 520px) {
    .lab-heading,
    .lab-panel { padding: 20px; }
    .lab-tabs { grid-template-columns: 1fr; }
    .lab-tabs button { border-right: 0; border-bottom: 1px solid var(--line); }
    .sft-controls { align-items: stretch; flex-direction: column; }
    .sft-controls select { width: 100%; margin: 8px 0 0; }
    .sft-sequence,
    .metric-row,
    .rm-controls,
    .recipe-contract,
    .update-flow { grid-template-columns: 1fr; }
    .preference-grid dl { grid-template-columns: 1fr 1fr; }
  }
</style>