wuyang
2882 lines
110 KiB
Plaintext
2882 lines
110 KiB
Plaintext
---
|
||
import BaseLayout from "@/layouts/BaseLayout.astro";
|
||
import LanguageModelHistoryLab from "@/components/LanguageModelHistoryLab.astro";
|
||
|
||
const toc = [
|
||
["00", "compass", "先拿到八张账"],
|
||
["01", "prediction", "语言为什么能被预测"],
|
||
["02", "chain-rule", "整句话怎样拆成小题"],
|
||
["03", "information", "熵、损失与困惑度"],
|
||
["04", "units", "字、词、子词与字节"],
|
||
["05", "ngram", "N-gram 的天才与局限"],
|
||
["06", "smoothing", "零概率与平滑"],
|
||
["07", "kneser-ney", "Kneser–Ney 的关键转弯"],
|
||
["08", "representations", "离散类别到连续向量"],
|
||
["09", "bengio", "神经概率语言模型"],
|
||
["10", "embeddings", "Word2Vec 与静态表示"],
|
||
["11", "recurrent", "RNN：把历史带在身上"],
|
||
["12", "gradients", "为什么遥远信息会消失"],
|
||
["13", "lstm", "LSTM 的受控记忆通路"],
|
||
["14", "rnnlm", "RNNLM 的能力与系统税"],
|
||
["15", "seq2seq", "从续写到序列转导"],
|
||
["16", "attention-bridge", "Attention：按步重新读取"],
|
||
["17", "economics", "词表、评测与计算"],
|
||
["18", "modern", "K3 / DeepSeek 的当代回声"],
|
||
["19", "synthesis", "一条完整因果链"],
|
||
["↳", "papers", "33 个关键节点"],
|
||
];
|
||
|
||
const ledgers = [
|
||
{ id: "UNIT", title: "预测单位", question: "一次预测什么？", path: "字母 → 词 → 子词 → 字节", trap: "Token 不是天然语言单位" },
|
||
{ id: "INFO", title: "概率与信息", question: "怎样衡量猜得好？", path: "链式法则 → NLL → PPL", trap: "困惑度不等于理解力" },
|
||
{ id: "CONTEXT", title: "上下文", question: "条件分布看多远？", path: "Markov → 窗口 → 状态", trap: "可访问不等于记得住" },
|
||
{ id: "SPARSE", title: "稀疏性", question: "没见过怎么办？", path: "折扣 → 回退 → 插值", trap: "平滑不只是“加一”" },
|
||
{ id: "REP", title: "分布式表示", question: "相似词怎样共享？", path: "类别 → 矩阵 → Embedding", trap: "静态向量不懂语境" },
|
||
{ id: "MEMORY", title: "循环记忆", question: "历史怎样穿过时间？", path: "SRN → 梯度 → LSTM / GRU", trap: "门控不等于无限记忆" },
|
||
{ id: "SEQ", title: "序列转导", question: "怎样条件生成另一序列？", path: "Encoder → 瓶颈 → Attention", trap: "对齐权重不等于解释" },
|
||
{ id: "COST", title: "评测与成本", question: "什么路线真的跑得动？", path: "Softmax → Benchmark → 并行", trap: "只看架构会漏掉系统因果" },
|
||
];
|
||
|
||
const eras = [
|
||
{ year: "1948–1951", label: "概率语言观", detail: "Shannon 把语言视作带上下文的随机符号源，预测与信息量连在一起。" },
|
||
{ year: "1953–2001", label: "计数模型", detail: "N-gram 让估计可行；Good–Turing、Katz、Kneser–Ney 修补稀疏性。" },
|
||
{ year: "1990–2013", label: "分布与循环", detail: "连续表示跨相似词共享参数，RNN 用固定状态携带变长历史。" },
|
||
{ year: "1997–2014", label: "受控记忆", detail: "长程梯度困难被显式化，LSTM / GRU 为状态提供门控通路。" },
|
||
{ year: "2014–2015", label: "条件生成", detail: "Seq2Seq 统一变长映射，也把固定向量瓶颈暴露到台前。" },
|
||
{ year: "2014–2017", label: "可微检索", detail: "Attention 让 decoder 每一步重读源序列，最终从桥梁变成主体。" },
|
||
{ year: "2017 →", label: "规模化主干", detail: "Transformer 接过条件分布计算；next-token 目标仍持续到 K3 与 DeepSeek。" },
|
||
];
|
||
|
||
const keyPapers = [
|
||
{ year: 1948, title: "A Mathematical Theory of Communication", who: "Shannon", url: "https://doi.org/10.1002/j.1538-7305.1948.tb01338.x", role: "熵与随机源", tier: "主干" },
|
||
{ year: 1951, title: "Prediction and Entropy of Printed English", who: "Shannon", url: "https://doi.org/10.1002/j.1538-7305.1951.tb01366.x", role: "已知前文预测下一字母", tier: "主干" },
|
||
{ year: 1953, title: "The Population Frequencies of Species…", who: "Good", url: "https://doi.org/10.1093/biomet/40.3-4.237", role: "未见概率质量", tier: "主干" },
|
||
{ year: 1977, title: "Perplexity—a measure of the difficulty…", who: "Jelinek et al.", url: "https://doi.org/10.1121/1.2016299", role: "困惑度术语史", tier: "地图" },
|
||
{ year: 1987, title: "Estimation of Probabilities from Sparse Data…", who: "Katz", url: "https://ieeexplore.ieee.org/document/1165125", role: "折扣与回退", tier: "主干" },
|
||
{ year: 1990, title: "Finding Structure in Time", who: "Elman", url: "https://doi.org/10.1207/s15516709cog1402_1", role: "简单循环状态", tier: "主干" },
|
||
{ year: 1990, title: "Indexing by Latent Semantic Analysis", who: "Deerwester et al.", url: "https://doi.org/10.1002/(SICI)1097-4571(199009)41:6%3C391::AID-ASI1%3E3.0.CO;2-9", role: "全局低秩语义空间", tier: "地图" },
|
||
{ year: 1992, title: "Class-Based n-gram Models of Natural Language", who: "Brown et al.", url: "https://aclanthology.org/J92-4003/", role: "离散类别共享", tier: "主干" },
|
||
{ year: 1994, title: "Learning long-term dependencies with gradient descent is difficult", who: "Bengio et al.", url: "https://ieeexplore.ieee.org/document/279181", role: "长程梯度困难", tier: "主干" },
|
||
{ year: 1995, title: "Improved Backing-off for M-gram Language Modeling", who: "Kneser & Ney", url: "https://doi.org/10.1109/ICASSP.1995.479394", role: "续接概率", tier: "主干" },
|
||
{ year: 1996, title: "An Empirical Study of Smoothing Techniques…", who: "Chen & Goodman", url: "https://aclanthology.org/P96-1041/", role: "平滑受控比较", tier: "地图" },
|
||
{ year: 1997, title: "Long Short-Term Memory", who: "Hochreiter & Schmidhuber", url: "https://doi.org/10.1162/neco.1997.9.8.1735", role: "受控误差通路", tier: "主干" },
|
||
{ year: 2001, title: "A Bit of Progress in Language Modeling", who: "Goodman", url: "https://arxiv.org/abs/cs/0108005", role: "计数技术组合", tier: "地图" },
|
||
{ year: 2003, title: "A Neural Probabilistic Language Model", who: "Bengio et al.", url: "https://www.jmlr.org/papers/v3/bengio03a.html", role: "表示与条件分布联合学习", tier: "主干" },
|
||
{ year: 2005, title: "Hierarchical Probabilistic Neural Network Language Model", who: "Morin & Bengio", url: "https://proceedings.mlr.press/r5/morin05a.html", role: "层次输出树", tier: "主干" },
|
||
{ year: 2010, title: "Recurrent neural network based language model", who: "Mikolov et al.", url: "https://www.isca-archive.org/interspeech_2010/mikolov10_interspeech.html", role: "实用 RNNLM", tier: "主干" },
|
||
{ year: 2013, title: "Efficient Estimation of Word Representations in Vector Space", who: "Mikolov et al.", url: "https://arxiv.org/abs/1301.3781", role: "CBOW / Skip-gram", tier: "主干" },
|
||
{ year: 2013, title: "Distributed Representations of Words and Phrases…", who: "Mikolov et al.", url: "https://proceedings.neurips.cc/paper/2013/hash/9aa42b31882ec039965f3c4923ce901b-Abstract.html", role: "负采样、子采样与短语", tier: "地图" },
|
||
{ year: 2013, title: "One Billion Word Benchmark…", who: "Chelba et al.", url: "https://arxiv.org/abs/1312.3005", role: "大规模可比 LM 基准", tier: "地图" },
|
||
{ year: 2014, title: "GloVe: Global Vectors for Word Representation", who: "Pennington et al.", url: "https://aclanthology.org/D14-1162/", role: "全局共现向量", tier: "地图" },
|
||
{ year: 2014, title: "Learning Phrase Representations using RNN Encoder–Decoder…", who: "Cho et al.", url: "https://aclanthology.org/D14-1179/", role: "GRU 与 encoder–decoder", tier: "主干" },
|
||
{ year: 2014, title: "Sequence to Sequence Learning with Neural Networks", who: "Sutskever et al.", url: "https://proceedings.neurips.cc/paper_files/paper/2014/hash/5a18e133cbf9f257297f410bb7eca942-Abstract.html", role: "深层 LSTM Seq2Seq", tier: "主干" },
|
||
{ year: 2014, title: "Neural Machine Translation by Jointly Learning to Align and Translate", who: "Bahdanau et al.", url: "https://arxiv.org/abs/1409.0473", role: "软对齐打破固定向量", tier: "主干" },
|
||
{ year: 2015, title: "Effective Approaches to Attention-based NMT", who: "Luong et al.", url: "https://aclanthology.org/D15-1166/", role: "global / local attention", tier: "地图" },
|
||
{ year: 2016, title: "Neural Machine Translation of Rare Words with Subword Units", who: "Sennrich et al.", url: "https://aclanthology.org/P16-1162/", role: "BPE 子词路线", tier: "主干" },
|
||
{ year: 2016, title: "Character-Aware Neural Language Models", who: "Kim et al.", url: "https://arxiv.org/abs/1508.06615", role: "字符组合词表示", tier: "地图" },
|
||
{ year: 2016, title: "Exploring the Limits of Language Modeling", who: "Jozefowicz et al.", url: "https://arxiv.org/abs/1602.02410", role: "大规模 RNNLM", tier: "地图" },
|
||
{ year: 2016, title: "Efficient softmax approximation for GPUs", who: "Grave et al.", url: "https://arxiv.org/abs/1609.04309", role: "Adaptive Softmax", tier: "地图" },
|
||
{ year: 2016, title: "Language Modeling with Gated Convolutional Networks", who: "Dauphin et al.", url: "https://arxiv.org/abs/1612.08083", role: "并行卷积路线", tier: "地图" },
|
||
{ year: 2017, title: "Regularizing and Optimizing LSTM Language Models", who: "Merity et al.", url: "https://arxiv.org/abs/1708.02182", role: "AWD-LSTM 强基线", tier: "地图" },
|
||
{ year: 2017, title: "Attention Is All You Need", who: "Vaswani et al.", url: "https://arxiv.org/abs/1706.03762", role: "本章终点 / 下一章起点", tier: "主干" },
|
||
{ year: 2024, title: "DeepSeek-V3 Technical Report", who: "DeepSeek-AI", url: "https://arxiv.org/abs/2412.19437", role: "NTP + 顺序 MTP", tier: "当代" },
|
||
{ year: 2026, title: "Kimi K3 Technical Report", who: "Moonshot AI", url: "https://www.kimi.com/blog/kimi-k3", role: "统一多模态 NTP + draft bridge", tier: "当代" },
|
||
];
|
||
---
|
||
|
||
<BaseLayout
|
||
title="语言模型从哪里来：从 Shannon、N-gram 到 Attention 前夜"
|
||
description="用八张账、33 个一手节点和四个交互实验，从零理解语言概率、N-gram、平滑、词向量、RNN/LSTM、Seq2Seq、Attention，以及 Kimi K3 与 DeepSeek 的 next-token 继承关系。"
|
||
section="foundations"
|
||
>
|
||
<header class="page-hero lm-hero">
|
||
<div class="page-hero-inner">
|
||
<div>
|
||
<p class="eyebrow"><span>FOUNDATIONS / 01</span> LANGUAGE MODELING</p>
|
||
<h1>今天的大模型，始于一个<br />看似朴素的问题</h1>
|
||
<p class="hero-question">“知道前文以后，下一个符号有多难猜？”</p>
|
||
<p class="lead">
|
||
从 Shannon 的猜字实验到 K3 的 160K 词表与统一多模态预测，
|
||
主问题没有消失。真正变化的是：预测单位、上下文函数、参数共享、记忆路径和可承受的计算成本。
|
||
</p>
|
||
</div>
|
||
<dl class="page-facts">
|
||
<div><dt>LEVEL</dt><dd>L0 直觉 → L3 论文</dd></div>
|
||
<div><dt>LINEAGE</dt><dd>1948 → 2026</dd></div>
|
||
<div><dt>PRIMARY NODES</dt><dd>33 个一手节点</dd></div>
|
||
<div><dt>INTERACTIVE</dt><dd>四联机制实验</dd></div>
|
||
<div><dt>READING</dt><dd>约 150–190 分钟</dd></div>
|
||
<div><dt>STATUS</dt><dd>深度首版 · 继续逐图精读</dd></div>
|
||
</dl>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="report-shell">
|
||
<aside class="side-rail" aria-label="本页目录">
|
||
<p>CONTENTS</p>
|
||
<ol>
|
||
{toc.map(([number, id, label]) => (
|
||
<li><a href={`#${id}`}><span>{number}</span>{label}</a></li>
|
||
))}
|
||
</ol>
|
||
<div class="rail-note">
|
||
<b>第一次阅读</b>
|
||
只追踪每节的“旧墙 → 新桥 → 新问题”。公式先看符号角色，不必先会求导。
|
||
</div>
|
||
<div class="rail-note evidence-note">
|
||
<b>证据图例</b>
|
||
<span><i class="verified"></i>原论文明确内容</span>
|
||
<span><i class="explain"></i>教学解释</span>
|
||
<span><i class="inference"></i>跨时代推论</span>
|
||
</div>
|
||
</aside>
|
||
|
||
<article class="article">
|
||
<section class="article-section" id="compass">
|
||
<p class="eyebrow"><span>00</span> EIGHT LEDGERS</p>
|
||
<h2>先不要背模型名：拿到八张彼此独立的账</h2>
|
||
<p class="lede">
|
||
一项语言模型技术通常只改了一两张账。把它们混在一起，就会得到“Transformer 比 RNN 更聪明”
|
||
这类无法检验的叙述；把账分开，才能准确说出进步来自哪里。
|
||
</p>
|
||
|
||
<div class="ledger-grid">
|
||
{ledgers.map((ledger, index) => (
|
||
<article>
|
||
<span>{String(index + 1).padStart(2, "0")} / {ledger.id}</span>
|
||
<h3>{ledger.title}</h3>
|
||
<b>{ledger.question}</b>
|
||
<p>{ledger.path}</p>
|
||
<small>易错：{ledger.trap}</small>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<div class="reading-contract">
|
||
<div>
|
||
<span>FACT / 事实</span>
|
||
<p>原论文直接写出的机制、公式、实验与限制；数字必须连同设置一起出现。</p>
|
||
</div>
|
||
<div>
|
||
<span>EXPLANATION / 解释</span>
|
||
<p>为降低门槛做的比喻和简化；必须说明它省略了什么。</p>
|
||
</div>
|
||
<div>
|
||
<span>INFERENCE / 推论</span>
|
||
<p>把旧论文放进 K3 / DeepSeek 语境后的跨时代连接，不伪装成作者原话。</p>
|
||
</div>
|
||
</div>
|
||
|
||
<h3>先看全局年代，不把它误读成单线接力</h3>
|
||
<div class="era-map">
|
||
{eras.map((era, index) => (
|
||
<div>
|
||
<span>{era.year}</span>
|
||
<i>{String(index + 1).padStart(2, "0")}</i>
|
||
<article><h4>{era.label}</h4><p>{era.detail}</p></article>
|
||
</div>
|
||
))}
|
||
</div>
|
||
<p class="figure-note">
|
||
<b>读图边界</b> 多条路线长期并存：强平滑 N-gram、RNN、卷积 LM 在 Transformer 出现后仍有实际价值。
|
||
这张图是问题链，不是“后一篇彻底消灭前一篇”的胜者史。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="prediction">
|
||
<p class="eyebrow"><span>01</span> PREDICTION AS A LENS</p>
|
||
<h2>语言为什么能被预测：因为选择不是均匀的</h2>
|
||
<p>
|
||
如果每个下一个符号都完全随机，前文就没有用。真实语言恰好相反：
|
||
中文里“人工智”之后，“能”比“柜”合理得多；英语里 Q 后面常跟 U；
|
||
代码里的左括号、变量作用域、函数签名又提供更硬的约束。
|
||
</p>
|
||
<div class="prediction-demo">
|
||
<div class="context-strip">
|
||
<span>CONTEXT</span>
|
||
<b>今天的天气很</b>
|
||
</div>
|
||
<div class="candidate-list">
|
||
<div><span>好</span><i style="width:78%"></i><b>0.46</b></div>
|
||
<div><span>冷</span><i style="width:55%"></i><b>0.25</b></div>
|
||
<div><span>热</span><i style="width:38%"></i><b>0.14</b></div>
|
||
<div><span>复杂</span><i style="width:16%"></i><b>0.04</b></div>
|
||
</div>
|
||
<p>示意概率。语言模型不是先挑一个词，而是先给整个候选集合分配概率。</p>
|
||
</div>
|
||
|
||
<h3>Shannon 1951 做的不是现代 LLM，却抓住了同一个可测问题</h3>
|
||
<p>
|
||
Shannon 让预测者在已知 preceding text 的条件下猜 next letter，再用猜测顺序估计英语的熵与冗余。
|
||
上下文越长，预测通常越容易，说明语言含有跨字符、单词乃至句子的统计约束。
|
||
</p>
|
||
<div class="source-boundary">
|
||
<span>PRIMARY SOURCE</span>
|
||
<div>
|
||
<h4>Prediction and Entropy of Printed English · 1951</h4>
|
||
<p>论文直接支持：下一字母预测、上下文、熵与冗余。它不支持“Shannon 发明了神经语言模型”。</p>
|
||
</div>
|
||
<a href="https://doi.org/10.1002/j.1538-7305.1951.tb01366.x" target="_blank" rel="noreferrer">Wiley / DOI ↗</a>
|
||
</div>
|
||
|
||
<div class="concept-split">
|
||
<article><span>模型学到的直接对象</span><h3>条件概率</h3><p>在当前上下文下，每个候选 token 有多可能。</p></article>
|
||
<article><span>为降低误差可能形成的内部结构</span><h3>语法 · 事实 · 模式</h3><p>这些规律能提高预测，因此被压进参数与激活。</p></article>
|
||
<article><span>目标没有自动保证的东西</span><h3>真实 · 忠实 · 对齐</h3><p>高似然不等于事实正确，合理解释也不保证是实际因果过程。</p></article>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="chain-rule">
|
||
<p class="eyebrow"><span>02</span> CHAIN RULE</p>
|
||
<h2>一整句话的概率，可以拆成一串“下一步”</h2>
|
||
<p>
|
||
联合概率看似要一次理解所有词之间的关系。链式法则把它拆成许多条件概率：
|
||
先问第一个 token，再问已知第一个时第二个，再问已知前两个时第三个。
|
||
</p>
|
||
<div class="formula large">
|
||
P(x₁, x₂, …, xₜ) = P(x₁) · P(x₂|x₁) · … · P(xₜ|x<ₜ)
|
||
<small>这是概率恒等式，不是 Transformer 专属公式，也不要求神经网络。</small>
|
||
</div>
|
||
<div class="chain-visual" role="img" aria-label="联合概率按链式法则拆成逐步条件概率">
|
||
<div><span>01</span><b>我</b><em>P(我)</em></div>
|
||
<i>×</i>
|
||
<div><span>02</span><b>喜欢</b><em>P(喜欢 | 我)</em></div>
|
||
<i>×</i>
|
||
<div><span>03</span><b>读</b><em>P(读 | 我 喜欢)</em></div>
|
||
<i>×</i>
|
||
<div><span>04</span><b>论文</b><em>P(论文 | 我 喜欢 读)</em></div>
|
||
</div>
|
||
|
||
<h3>三个经常混在一起的选择</h3>
|
||
<div class="three-choices">
|
||
<div><span>A / MATHEMATICS</span><h4>怎样分解联合概率</h4><p>链式法则允许任意顺序；从左到右只是其中一种。</p></div>
|
||
<div><span>B / OBJECTIVE</span><h4>训练预测哪一侧</h4><p>因果 LM 选前缀预测下一个；BERT masked LM 选择遮住位置。</p></div>
|
||
<div><span>C / ARCHITECTURE</span><h4>条件分布怎样计算</h4><p>N-gram 查表、RNN 状态、Transformer attention 都可服务同一目标。</p></div>
|
||
</div>
|
||
|
||
<p>
|
||
因此，“next-token prediction”不是一个具体架构。它是一种概率分解与训练题型；
|
||
2017 年之后真正改变的是计算 \(P(x_t|x_<t)\) 的上下文函数，以及训练它所能承受的数据和计算规模。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="information">
|
||
<p class="eyebrow"><span>03</span> INFORMATION & METRICS</p>
|
||
<h2>猜错多少，不用“像不像人”来量：用 surprise、NLL 与 PPL</h2>
|
||
|
||
<div class="metric-ladder">
|
||
<article>
|
||
<span>01 / SURPRISE</span>
|
||
<div class="formula compact">I(x) = −log P(x)</div>
|
||
<p>正确答案概率越低，看到它时越“意外”，惩罚越大。</p>
|
||
</article>
|
||
<article>
|
||
<span>02 / NLL</span>
|
||
<div class="formula compact">L = −Σₜ log Pθ(xₜ|x<ₜ)</div>
|
||
<p>把每个正确 token 的 surprise 相加；最小化它等价于最大化序列似然。</p>
|
||
</article>
|
||
<article>
|
||
<span>03 / PERPLEXITY</span>
|
||
<div class="formula compact">PPL = exp(L / T)</div>
|
||
<p>把平均 NLL 指数化，回到“有效候选数”的直觉尺度。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<h3>手算一个两步例子</h3>
|
||
<div class="worked-example">
|
||
<div><span>模型 A</span><p>正确 token 概率：0.50、0.50</p><b>平均 NLL = 0.693</b><em>PPL = 2.00</em></div>
|
||
<div><span>模型 B</span><p>正确 token 概率：0.90、0.10</p><b>平均 NLL = 1.204</b><em>PPL = 3.33</em></div>
|
||
</div>
|
||
<p>
|
||
两个模型的平均正确概率看起来都可能“差不多”，但 log loss 会强烈惩罚第二个模型对某一步极度自信地猜错。
|
||
这正是乘法概率在 log 空间里变成加法后的效果。
|
||
</p>
|
||
|
||
<div class="warning-note">
|
||
<b>困惑度比较的四个闸门</b>
|
||
<p>
|
||
必须同测试集、同预处理、同词表/OOV 规则、最好同 Tokenizer。一个模型每个词切 1 个 token，
|
||
另一个切 3 个，原始 token PPL 不能直接当赛跑成绩。PPL 也不直接测事实性、指令遵循、推理可靠性或安全性。
|
||
</p>
|
||
</div>
|
||
|
||
<h3>为什么简单目标会诱导复杂能力</h3>
|
||
<p>
|
||
补全“牛顿在 1687 年出版了……”需要事实；补全函数体需要变量作用域和算法；
|
||
补全论证需要跟踪前提与结论。只要某种内部规律能在大量上下文中共同降低 NLL，
|
||
学到它就比逐句死记更经济。这解释“能力为何可能出现”，不构成“所有能力必然可靠”的保证。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="units">
|
||
<p class="eyebrow"><span>04</span> PREDICTION UNIT</p>
|
||
<h2>预测什么单位，会改变序列长度、词表税和跨语言边界</h2>
|
||
|
||
<div class="unit-comparison">
|
||
<article>
|
||
<span>CHARACTER</span><h3>字符</h3><p>词表小、无大规模 OOV；序列更长，一个语义单位要走更多步。</p>
|
||
<code>不 / 可 / 思 / 议</code>
|
||
</article>
|
||
<article>
|
||
<span>WORD</span><h3>词</h3><p>序列短、语义粒度直观；词表巨大，屈折、复合词和新词不断制造 OOV。</p>
|
||
<code>不可思议</code>
|
||
</article>
|
||
<article class="accent">
|
||
<span>SUBWORD</span><h3>子词</h3><p>固定词表覆盖开放文本；常见片段保持完整，稀有词退回更小单元。</p>
|
||
<code>不可 / 思议</code>
|
||
</article>
|
||
<article>
|
||
<span>BYTE</span><h3>字节</h3><p>覆盖任何 Unicode 字符串；语义单位更碎，长非拉丁文本可能付出额外长度。</p>
|
||
<code>E4 / B8 / 8D / …</code>
|
||
</article>
|
||
</div>
|
||
|
||
<h3>BPE 只做一件事：反复合并高频相邻单元</h3>
|
||
<div class="bpe-steps">
|
||
<div><span>0</span><code>l o w </w></code><p>从字符/字节开始</p></div>
|
||
<i>→</i>
|
||
<div><span>1</span><code>lo w </w></code><p>合并高频 l + o</p></div>
|
||
<i>→</i>
|
||
<div><span>2</span><code>low </w></code><p>继续合并 lo + w</p></div>
|
||
</div>
|
||
<p>
|
||
Sennrich 等人在 2016 年把 BPE 系统引入神经机器翻译的稀有词问题。
|
||
它是在固定词表与开放词汇之间做工程折中，不知道 morpheme，也不保证每次切分符合人类语义。
|
||
</p>
|
||
|
||
<div class="tokenizer-cost">
|
||
<div><span>词表变大</span><b>序列可能变短</b><p>但 embedding / LM head 更大，低频 token 学得少。</p></div>
|
||
<i>⇄</i>
|
||
<div><span>词表变小</span><b>覆盖与共享更强</b><p>但序列变长，attention / recurrence 要处理更多步。</p></div>
|
||
</div>
|
||
|
||
<div class="modern-note">
|
||
<span>回看当代</span>
|
||
<p>
|
||
DeepSeek-V3/V4 报告给出 128K byte-level BPE；K3 配置表给出 160K vocabulary。
|
||
词表规模不是“知道多少词”，而是模型在序列长度、输出计算、跨语言覆盖与训练频率之间做的接口选择。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="ngram">
|
||
<p class="eyebrow"><span>05</span> COUNTING CONTEXT</p>
|
||
<h2>N-gram 的天才：用有限后缀把不可能的统计变成可估计</h2>
|
||
<p>
|
||
完整前缀几乎永远不会原样重复。N-gram 做一个强但实用的 Markov 假设：
|
||
预测下一个词时，只看最近 \(n-1\) 个词。
|
||
</p>
|
||
<div class="formula">
|
||
P(wₜ | w₁, …, wₜ₋₁) ≈ P(wₜ | wₜ₋ₙ₊₁, …, wₜ₋₁)
|
||
<small>左边来自链式法则；右边的有限历史是建模假设。</small>
|
||
</div>
|
||
|
||
<div class="ngram-window">
|
||
<div><span>UNIGRAM</span><div><i>我</i><i>喜欢</i><i>读</i><b>?</b></div><p>不看上下文，只看候选总体频率。</p></div>
|
||
<div><span>BIGRAM</span><div><i>我</i><i>喜欢</i><em>读</em><b>?</b></div><p>只根据“读”估计下一词。</p></div>
|
||
<div><span>TRIGRAM</span><div><i>我</i><em>喜欢</em><em>读</em><b>?</b></div><p>根据“喜欢 读”估计下一词。</p></div>
|
||
</div>
|
||
|
||
<h3>最大似然估计只是“在这个上下文后出现了多少次”</h3>
|
||
<div class="formula">
|
||
P̂(w | h) = C(h, w) / C(h)
|
||
<small>例如语料里“喜欢 读”出现 10 次，其中 6 次后接“书”，则 MLE 为 0.6。</small>
|
||
</div>
|
||
|
||
<div class="tradeoff-axis">
|
||
<div><span>短上下文</span><b>统计稳定</b><p>见过很多次，但区分不了细节。</p></div>
|
||
<div class="axis-line"><i></i></div>
|
||
<div><span>长上下文</span><b>条件精确</b><p>更贴近当前句子，但几乎都没见过。</p></div>
|
||
</div>
|
||
|
||
<h3>组合为什么会爆炸</h3>
|
||
<p>
|
||
词表大小为 \(V\) 时，理论上可能的 N-gram 有 \(V^n\) 种。若 \(V=50,000\)，
|
||
仅 trigram 空间就有 \(1.25\times10^{14}\) 种；语料永远覆盖不了。
|
||
所以 N-gram 的核心问题不是表“放不下”这么简单，而是绝大多数格子永远没有可靠统计。
|
||
</p>
|
||
|
||
<div class="old-is-smart">
|
||
<span>不要低估计数模型</span>
|
||
<p>
|
||
类别模型、cache、skip-gram 特征、高阶回退和多个模型插值，让计数时代的强系统远比“数一个 bigram”复杂。
|
||
Goodman 2001 的组合实验正是提醒：技术史不是一条脆弱 baseline 被新模型轻松击倒的故事。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="smoothing">
|
||
<p class="eyebrow"><span>06</span> SPARSITY & SMOOTHING</p>
|
||
<h2>“没见过”如果等于“不可能”，一句话会被一个零击穿</h2>
|
||
<p>
|
||
测试句里只要出现一个训练集未见 N-gram，最大似然会给它概率 0；
|
||
乘法链让整句概率归零，NLL 变成无穷。这显然不符合开放语言。
|
||
</p>
|
||
|
||
<div class="zero-chain">
|
||
<div><b>0.42</b><span>P(我)</span></div><i>×</i>
|
||
<div><b>0.70</b><span>P(喜欢|我)</span></div><i>×</i>
|
||
<div><b>0.50</b><span>P(吃|喜欢)</span></div><i>×</i>
|
||
<div class="zero"><b>0.00</b><span>P(梨|吃)</span></div><i>=</i>
|
||
<strong>0</strong>
|
||
</div>
|
||
|
||
<h3>平滑的共同原则：从“过度自信的已见事件”挪一点质量</h3>
|
||
<div class="smoothing-pipeline">
|
||
<article><span>01 / DISCOUNT</span><h4>折扣</h4><p>把低计数事件的原始概率向下修，释放概率质量。</p></article>
|
||
<i>→</i>
|
||
<article><span>02 / LOWER ORDER</span><h4>低阶证据</h4><p>高阶组合不可靠时，查看更短上下文怎样分布。</p></article>
|
||
<i>→</i>
|
||
<article><span>03 / REDISTRIBUTE</span><h4>重新分配</h4><p>把释放的质量分给未见但合理的事件，保持总和为 1。</p></article>
|
||
</div>
|
||
|
||
<h3>Good–Turing：未见质量藏在“只出现一次的种类有多少”里</h3>
|
||
<p>
|
||
Good 1953 研究物种总体频率，也明确提到文学词汇。它关注 frequency of frequencies：
|
||
出现 1 次的不同事件有多少、出现 2 次的有多少。大量 singleton 意味着采样仍在不断遇到新类型，
|
||
未见总体概率质量就不该被设成 0。
|
||
</p>
|
||
|
||
<h3>Katz Backoff：高阶有证据就用高阶，没有就回退</h3>
|
||
<div class="backoff-tree">
|
||
<div><span>TRIGRAM</span><b>“喜欢 吃” → 梨</b><em>见过且计数可靠？</em></div>
|
||
<div class="branch yes"><span>YES</span><b>折扣后的高阶概率</b><p>保留具体上下文优势</p></div>
|
||
<div class="branch no"><span>NO</span><b>回退到 “吃” → 梨</b><p>用低阶模型分配未见质量</p></div>
|
||
</div>
|
||
|
||
<div class="myth-grid">
|
||
<div><span>回退 Backoff</span><p>高阶缺失/不可靠时才转向低阶。</p></div>
|
||
<div><span>插值 Interpolation</span><p>不论高阶是否见过，都混合多阶概率。</p></div>
|
||
<div><span>Add-one</span><p>每个事件加同样伪计数，直观但常不是大词表 LM 的强方案。</p></div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="kneser-ney">
|
||
<p class="eyebrow"><span>07</span> CONTINUATION PROBABILITY</p>
|
||
<h2>Kneser–Ney 最漂亮的一步：低阶模型不该再数同一件事</h2>
|
||
<p>
|
||
普通回退常用一个词的总体频率做低阶概率。但高频不一定代表“在新上下文里容易出现”。
|
||
“Francisco”可能出现很多次，却几乎都跟在 “San” 后面；它不该因为总频率高，就在任意前词后都有高概率。
|
||
</p>
|
||
|
||
<div class="continuation-compare">
|
||
<article>
|
||
<span>ORDINARY UNIGRAM</span>
|
||
<h3>总共出现多少次？</h3>
|
||
<div class="count-bars">
|
||
<div><b>Francisco</b><i style="width:88%"></i><em>高频</em></div>
|
||
<div><b>Tuesday</b><i style="width:44%"></i><em>中频</em></div>
|
||
</div>
|
||
<p>会把大量 “San Francisco” 重复当成广泛适用证据。</p>
|
||
</article>
|
||
<article class="accent">
|
||
<span>KN CONTINUATION</span>
|
||
<h3>跟在多少种不同上下文后？</h3>
|
||
<div class="context-cloud">
|
||
<div><b>Francisco</b><code>San →</code></div>
|
||
<div><b>Tuesday</b><code>on / last / next / every / this →</code></div>
|
||
</div>
|
||
<p>Tuesday 的不同左上下文更多，因此更像一个通用续接词。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<div class="formula">
|
||
P<sub>continuation</sub>(w) ∝ N₁⁺(· w)
|
||
<small>N₁⁺(· w) 表示 w 出现过的不同左上下文类型数，而不是 w 的原始 token 总数。</small>
|
||
</div>
|
||
|
||
<p>
|
||
这不是一个小修补，而是一条建模原则：层次模型的低层不应机械重复高层证据，
|
||
而应提供高层缺失时真正互补的信息。后来的 modified interpolated Kneser–Ney
|
||
扩展折扣细节；不能把所有实用公式都倒算到 1995 原文。
|
||
</p>
|
||
|
||
<div class="evidence-row">
|
||
<a href="https://doi.org/10.1109/ICASSP.1995.479394" target="_blank" rel="noreferrer">
|
||
<span>1995 / ORIGIN</span><b>Kneser & Ney</b><em>Improved Backing-off ↗</em>
|
||
</a>
|
||
<a href="https://aclanthology.org/P96-1041/" target="_blank" rel="noreferrer">
|
||
<span>1996 / EMPIRICAL</span><b>Chen & Goodman</b><em>Smoothing comparison ↗</em>
|
||
</a>
|
||
<a href="https://arxiv.org/abs/cs/0108005" target="_blank" rel="noreferrer">
|
||
<span>2001 / SYSTEM</span><b>Goodman</b><em>Technique combination ↗</em>
|
||
</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="representations">
|
||
<p class="eyebrow"><span>08</span> PARAMETER SHARING</p>
|
||
<h2>计数模型只会精确匹配；语言需要“相似但不相同”也能借力</h2>
|
||
|
||
<div class="sharing-staircase">
|
||
<article><span>01</span><h3>词 ID</h3><p>只有完全相同的符号共享计数。</p><code>猫 ≠ 狗 ≠ 老虎</code></article>
|
||
<i>→</i>
|
||
<article><span>02</span><h3>离散词类</h3><p>Brown 1992 让同一类别内的词共享统计。</p><code>动物 = {猫, 狗, …}</code></article>
|
||
<i>→</i>
|
||
<article class="accent"><span>03</span><h3>连续向量</h3><p>相似性不再只有“同类/不同类”两档。</p><code>distance(猫, 狗) < distance(猫, 微积分)</code></article>
|
||
</div>
|
||
|
||
<h3>One-hot 没有语义距离</h3>
|
||
<p>
|
||
词表里每个词占一个正交轴。猫 `[1,0,0,…]` 与狗 `[0,1,0,…]` 的点积为 0；
|
||
猫与微积分也同样为 0。ID 的大小和轴的位置只是目录安排。
|
||
</p>
|
||
<div class="formula">
|
||
e<sub>one-hot</sub>(w) ∈ {0,1}<sup>|V|</sup>
|
||
<small>每个词一个维度，维数等于词表；不同词之间没有“更像一点”的表达空间。</small>
|
||
</div>
|
||
|
||
<h3>分布式表示把一个概念摊到许多维度</h3>
|
||
<p>
|
||
一个词用 \(d\) 个连续数表示；一个维度参与很多词，一个词也依赖很多维度。
|
||
这使得网络对“猫”学到的某些更新，可以通过相近表示影响“狗”，从而跨未见组合泛化。
|
||
</p>
|
||
|
||
<div class="representation-boundary">
|
||
<div><span>LSA / 1990</span><b>全局矩阵分解</b><p>词—文档共现经 SVD 得到低维空间；没有词序与 next-token 目标。</p></div>
|
||
<div><span>BROWN / 1992</span><b>离散类别</b><p>一个词通常归一个类别；仍是类 N-gram 的概率分解。</p></div>
|
||
<div><span>NEURAL LM / 2003</span><b>预测中联合学习</b><p>Embedding 的好坏由它是否帮助条件概率泛化来决定。</p></div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="bengio">
|
||
<p class="eyebrow"><span>09</span> NEURAL PROBABILISTIC LM</p>
|
||
<h2>Bengio 2003：把词典、表示和条件分布放进同一个可微系统</h2>
|
||
<p>
|
||
这篇论文的历史关键不只是“用了神经网络”，而是用 distributed representation
|
||
对抗离散序列的 curse of dimensionality：相似词的向量接近，替换一个词不再让整个上下文完全失联。
|
||
</p>
|
||
|
||
<div class="nplm-diagram" role="img" aria-label="Bengio 2003 神经概率语言模型原创重绘">
|
||
<div class="nplm-inputs">
|
||
<span>FIXED CONTEXT</span>
|
||
<div><b>猫</b><em>ID 81</em></div>
|
||
<div><b>坐在</b><em>ID 215</em></div>
|
||
<div><b>柔软的</b><em>ID 903</em></div>
|
||
</div>
|
||
<i>lookup</i>
|
||
<div class="nplm-embed"><span>SHARED TABLE C</span><code>e₁ ⊕ e₂ ⊕ e₃</code><small>查表并拼接</small></div>
|
||
<i>→</i>
|
||
<div class="nplm-hidden"><span>NONLINEAR</span><b>tanh hidden</b><small>学习上下文特征</small></div>
|
||
<i>→</i>
|
||
<div class="nplm-output"><span>VOCABULARY</span><b>Softmax</b><small>P(下一个词 | 前 3 词)</small></div>
|
||
</div>
|
||
|
||
<h3>端到端训练发生了什么</h3>
|
||
<ol class="numbered-explanation">
|
||
<li><span>1</span><p>上下文词通过同一张 embedding table 查向量，因此参数会跨所有出现位置复用。</p></li>
|
||
<li><span>2</span><p>前馈网络把固定窗口映射到全词表 logits，并对正确下一词计算 NLL。</p></li>
|
||
<li><span>3</span><p>梯度不仅更新网络，也更新上下文词向量；能互相替换的词受到相似训练压力。</p></li>
|
||
<li><span>4</span><p>未见过的完整 N-gram 只要由相似词构成，仍可能得到合理概率，而不是只能精确回退。</p></li>
|
||
</ol>
|
||
|
||
<div class="bottleneck-grid">
|
||
<article><span>SOLVED</span><h3>离散共享</h3><p>连续表示让相似词和相似上下文共享统计强度。</p></article>
|
||
<article><span>LEFT OPEN</span><h3>固定窗口</h3><p>看多远仍由输入长度预先规定，窗口外信息完全不可见。</p></article>
|
||
<article><span>LEFT OPEN</span><h3>全词表 Softmax</h3><p>每个样本给所有词打分，训练与推断都贵。</p></article>
|
||
</div>
|
||
|
||
<div class="source-boundary">
|
||
<span>PRIMARY SOURCE</span>
|
||
<div>
|
||
<h4>A Neural Probabilistic Language Model · JMLR 2003</h4>
|
||
<p>本章固定引用四作者 JMLR 版本；不把它与 NIPS 2000 前身的作者列表混写。</p>
|
||
</div>
|
||
<a href="https://www.jmlr.org/papers/v3/bengio03a.html" target="_blank" rel="noreferrer">JMLR ↗</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="embeddings">
|
||
<p class="eyebrow"><span>10</span> STATIC WORD VECTORS</p>
|
||
<h2>Word2Vec 把表示学习做得极高效，但它不是完整语言模型</h2>
|
||
<div class="word2vec-compare">
|
||
<article>
|
||
<span>CBOW</span>
|
||
<div class="window-diagram"><i>今天</i><i>天气</i><b>?</b><i>适合</i><i>散步</i></div>
|
||
<h3>上下文 → 中心词</h3>
|
||
<p>把窗口内上下文聚合，预测被遮在中间的词。</p>
|
||
</article>
|
||
<article>
|
||
<span>SKIP-GRAM</span>
|
||
<div class="window-diagram"><i>今天</i><i>天气</i><b>很好</b><i>适合</i><i>散步</i></div>
|
||
<h3>中心词 → 周围词</h3>
|
||
<p>给定中心词，预测窗口里的多个邻近词。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<h3>Negative Sampling 改了训练题</h3>
|
||
<p>
|
||
与其每次归一化整个词表，它把观察到的词对当正样本，再抽少量噪声词当负样本，做二分类。
|
||
这使大规模向量训练非常快，但得到的是有用表示目标，不是严格归一化的句子概率模型。
|
||
</p>
|
||
<div class="negative-sampling">
|
||
<div class="positive"><span>POSITIVE</span><b>猫 ↔ 喵</b><em>提高相容分数</em></div>
|
||
<div><span>NEGATIVE</span><b>猫 ↔ 微积分</b><em>降低相容分数</em></div>
|
||
<div><span>NEGATIVE</span><b>猫 ↔ 发票</b><em>降低相容分数</em></div>
|
||
</div>
|
||
|
||
<h3>类比为什么能出现，又为什么不能神化</h3>
|
||
<p>
|
||
某些语义/语法关系会在向量空间形成近似稳定方向，于是出现
|
||
`king − man + woman ≈ queen` 一类现象。它是数据、目标与几何共同形成的经验结构，
|
||
不是一个保证所有关系都满足的符号代数系统。
|
||
</p>
|
||
|
||
<div class="static-limit">
|
||
<span>STATIC EMBEDDING LIMIT</span>
|
||
<div><b>“苹果发布手机”</b><code>苹果 → 同一个向量</code></div>
|
||
<div><b>“我吃了苹果”</b><code>苹果 → 仍是同一个向量</code></div>
|
||
<p>静态词向量把多义词的用法混在一个点里；上下文化表示要等网络根据句子动态改写它。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="recurrent">
|
||
<p class="eyebrow"><span>11</span> RECURRENT STATE</p>
|
||
<h2>RNN：不再规定“只能看最近 N 个词”，而是把历史滚进一个状态</h2>
|
||
<div class="formula">
|
||
hₜ = fθ(hₜ₋₁, e(xₜ)) · P(xₜ₊₁|x≤ₜ) = softmax(Whₜ)
|
||
<small>同一组参数在每个时间步重复使用；状态长度固定，序列长度可变。</small>
|
||
</div>
|
||
|
||
<div class="rnn-unroll">
|
||
<div><span>x₁</span><b>我</b><i>↓</i><strong>h₁</strong></div>
|
||
<em>→</em>
|
||
<div><span>x₂</span><b>喜欢</b><i>↓</i><strong>h₂</strong></div>
|
||
<em>→</em>
|
||
<div><span>x₃</span><b>读</b><i>↓</i><strong>h₃</strong></div>
|
||
<em>→</em>
|
||
<div><span>x₄</span><b>论文</b><i>↓</i><strong>h₄</strong></div>
|
||
</div>
|
||
|
||
<h3>Elman 1990 的关键直觉</h3>
|
||
<p>
|
||
context units 把上一时刻 hidden activation 带回网络。隐藏状态不需要人工标注“名词”“从句边界”，
|
||
只要这些结构能帮助预测后续，训练就可能让状态组织出相应簇。
|
||
</p>
|
||
|
||
<div class="fixed-window-vs-state">
|
||
<article><span>FIXED WINDOW</span><h3>显式保留最近 3 个词</h3><p>窗口外完全不可见；窗口内信息不压缩。</p><code>[xₜ₋₂, xₜ₋₁, xₜ]</code></article>
|
||
<article><span>RECURRENT STATE</span><h3>把全部前缀压进 hₜ</h3><p>理论上接触过全部历史；细节必须共享有限状态容量。</p><code>hₜ = fold(x₁…xₜ)</code></article>
|
||
</div>
|
||
|
||
<p>
|
||
“RNN 使用任意长度上下文”只表示结构没有硬编码窗口上限，不表示任意遥远信息在数值上仍可恢复。
|
||
它把一个问题解决成了另一个问题：现在我们要问，状态更新能否在很多步后保留信号，以及梯度能否回到很早的位置。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="gradients">
|
||
<p class="eyebrow"><span>12</span> LONG-TERM DEPENDENCIES</p>
|
||
<h2>遥远信息为什么会消失：前向信号和反向梯度都要反复过同一扇门</h2>
|
||
<p>
|
||
训练 RNN 时，误差必须沿展开的时间图反向传播。跨 \(k\) 步的影响包含一串 Jacobian 矩阵乘积：
|
||
</p>
|
||
<div class="formula large">
|
||
∂hₜ / ∂hₜ₋ₖ = ∏ᵢ ∂hᵢ / ∂hᵢ₋₁
|
||
<small>教学标量版：若每步保留 0.9，20 步后只剩 0.9²⁰ ≈ 12.2%。</small>
|
||
</div>
|
||
|
||
<div class="gradient-regimes">
|
||
<article>
|
||
<span>|λ| < 1</span><h3>消失</h3>
|
||
<div class="decay-bars"><i style="width:100%"></i><i style="width:64%"></i><i style="width:28%"></i><i style="width:8%"></i></div>
|
||
<p>遥远位置几乎收不到学习信号。</p>
|
||
</article>
|
||
<article>
|
||
<span>|λ| ≈ 1</span><h3>保持</h3>
|
||
<div class="decay-bars stable"><i></i><i></i><i></i><i></i></div>
|
||
<p>需要一条接近恒等、又能受控读写的通路。</p>
|
||
</article>
|
||
<article>
|
||
<span>|λ| > 1</span><h3>爆炸</h3>
|
||
<div class="decay-bars explode"><i></i><i></i><i></i><i></i></div>
|
||
<p>数值快速放大，训练不稳定；gradient clipping 只能限制结果。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<h3>困难不只是激活函数“斜率小”</h3>
|
||
<p>
|
||
Bengio、Simard 与 Frasconi 1994 讨论的是更深的动力学与优化矛盾：
|
||
系统既要让状态对扰动稳定，才能长时间保存；又要让梯度足够敏感，才能学会何时改变状态。
|
||
把它压缩成一句“sigmoid 导致 vanishing gradient”，会丢失问题的结构。
|
||
</p>
|
||
|
||
<div class="distance-story">
|
||
<span>“我在法国出生，后来辗转许多国家……所以我的母语是 ___。”</span>
|
||
<div><b>线索</b><i></i><i></i><i></i><i></i><i></i><i></i><i></i><strong>预测</strong></div>
|
||
<p>线索必须前向穿过每个中间状态；训练误差又要沿相反方向走回去。步数就是最短路径长度。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="lstm">
|
||
<p class="eyebrow"><span>13</span> GATED MEMORY</p>
|
||
<h2>LSTM 的关键不是“记忆更大”，而是提供一条受控的近恒等通路</h2>
|
||
|
||
<div class="lstm-diagram" role="img" aria-label="现代教学版 LSTM 记忆通路">
|
||
<div class="cell-highway">
|
||
<span>cₜ₋₁</span><i>──────────────</i><b>× fₜ</b><i>────</i><strong>＋</strong><i>────</i><span>cₜ</span>
|
||
</div>
|
||
<div class="gate-row">
|
||
<div><span>FORGET</span><b>fₜ</b><p>旧记忆保留多少</p></div>
|
||
<div><span>INPUT</span><b>iₜ · c̃ₜ</b><p>新内容写入多少</p></div>
|
||
<div><span>OUTPUT</span><b>oₜ</b><p>当前状态读出多少</p></div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="formula">
|
||
cₜ = fₜ ⊙ cₜ₋₁ + iₜ ⊙ c̃ₜ
|
||
<small>如果一段时间内 fₜ 接近 1、写入很小，cell state 和梯度可以较平缓地穿过许多步。</small>
|
||
</div>
|
||
|
||
<h3>“门”不是开关，而是 0 到 1 的连续阀门</h3>
|
||
<p>
|
||
门值由当前输入和旧隐藏状态共同计算。模型可以在数字序列里长时间保留一个标志，
|
||
遇到分隔符时清空，再在需要输出时读出。它把状态更新从“每步全盘重写”变成“选择性保留、写入和暴露”。
|
||
</p>
|
||
|
||
<div class="history-correction">
|
||
<span>HISTORY CORRECTION</span>
|
||
<div>
|
||
<h3>1997 原版 ≠ 教科书三门默认版</h3>
|
||
<p>
|
||
Hochreiter–Schmidhuber 1997 原始 LSTM 强调 constant error carousel 与输入/输出门；
|
||
现代常画的 forget gate 是后继工作加入。为了讲机制，本页画现代教学版，但不把全部结构错误归给 1997。
|
||
</p>
|
||
</div>
|
||
</div>
|
||
|
||
<h3>LSTM 仍然没有解决什么</h3>
|
||
<div class="still-open">
|
||
<div><span>SEQUENTIAL</span><b>时间步仍串行</b><p>hₜ 依赖 hₜ₋₁，训练难把所有位置一次摊平。</p></div>
|
||
<div><span>CAPACITY</span><b>状态仍固定大小</b><p>很长历史仍竞争有限 cell / hidden 维度。</p></div>
|
||
<div><span>PATH</span><b>远距离仍需多步</b><p>门控改善信号，却没有建立任意位置间的一层直达边。</p></div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="rnnlm">
|
||
<p class="eyebrow"><span>14</span> PRACTICAL RNN LANGUAGE MODELS</p>
|
||
<h2>RNNLM 证明变长状态有真实收益，也把计算瓶颈摆上桌面</h2>
|
||
<p>
|
||
Mikolov 等 2010 把简单 RNNLM 放进语音识别任务。论文在其模型组合与任务设置下，
|
||
报告对强 backoff LM 约 50% 的 PPL 降低，以及 WSJ / NIST RT05 上约 18% / 5% 的相对 WER 降低；
|
||
同时明确指出高训练复杂度。
|
||
</p>
|
||
|
||
<div class="claim-contract">
|
||
<article><span>论文直接报告</span><h3>特定设置下的相对改善</h3><p>数据、混合方式、解码器与 baseline 都属于结论的一部分。</p></article>
|
||
<article><span>合理解释</span><h3>状态捕获了固定 N-gram 以外的互补信息</h3><p>cache / count model 与 RNN 仍可互补。</p></article>
|
||
<article><span>不能推出</span><h3>所有 RNN 都比所有 N-gram 好 50%</h3><p>作者数字不是跨数据、跨实现的通用常数。</p></article>
|
||
</div>
|
||
|
||
<h3>输出层为什么会成为税</h3>
|
||
<p>
|
||
隐藏状态算完以后，还要对词表每个词产生 logit 并归一化。
|
||
词表有十万词，就要处理十万个候选；模型主体变快，不代表输出层免费。
|
||
</p>
|
||
|
||
<div class="softmax-routes">
|
||
<article><span>FULL SOFTMAX</span><div class="vocab-wall">{Array.from({ length: 36 }).map(() => <i></i>)}</div><p>每步扫描整个词表；分布严格归一化。</p></article>
|
||
<article><span>HIERARCHICAL</span><div class="tree-mini"><b></b><i></i><i></i><em></em><em></em><em></em><em></em></div><p>沿词树路径做二分类；成本与路径深度相关。</p></article>
|
||
<article><span>ADAPTIVE</span><div class="adaptive-mini"><b>高频短名单</b><i>中频簇</i><i>低频簇</i></div><p>按频率分配不同计算；降低期望成本。</p></article>
|
||
</div>
|
||
|
||
<h3>2016–2017：Transformer 前夜并不是“只有 RNN”</h3>
|
||
<p>
|
||
Gated convolutional LM 证明卷积的有限感受野和并行性可以很有竞争力；
|
||
AWD-LSTM 又用 DropConnect、NT-ASGD 等配方把 LSTM 做成强基线。
|
||
架构史真正比较的是质量、并行、内存、延迟和可扩展性的联立解。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="seq2seq">
|
||
<p class="eyebrow"><span>15</span> CONDITIONAL GENERATION</p>
|
||
<h2>从“续写一个序列”到“把一个序列变成另一个序列”</h2>
|
||
<div class="formula">
|
||
P(y | x) = ∏ₜ P(yₜ | y<ₜ, x)
|
||
<small>目标端仍做 next-token prediction，只是每一步额外条件于源序列 x。</small>
|
||
</div>
|
||
|
||
<div class="seq2seq-diagram" role="img" aria-label="固定向量 encoder decoder">
|
||
<div class="encoder-side">
|
||
<span>ENCODER</span>
|
||
<div><b>I</b><i>→</i><b>read</b><i>→</i><b>papers</b></div>
|
||
<small>逐步把源句滚入状态</small>
|
||
</div>
|
||
<strong>C</strong>
|
||
<div class="decoder-side">
|
||
<span>DECODER</span>
|
||
<div><b>我</b><i>→</i><b>读</b><i>→</i><b>论文</b></div>
|
||
<small>条件于同一个固定向量逐步生成</small>
|
||
</div>
|
||
</div>
|
||
|
||
<h3>Cho 2014 与 Sutskever 2014 共同确立了什么</h3>
|
||
<ul class="plain-list">
|
||
<li>输入和输出不需要同长度；</li>
|
||
<li>不必手写每一种局部对齐或短语转换规则；</li>
|
||
<li>encoder 学表示，decoder 学条件语言模型，全部端到端训练；</li>
|
||
<li>GRU / LSTM 的门控状态帮助跨变长序列传播。</li>
|
||
</ul>
|
||
|
||
<h3>反转源序列为什么有效</h3>
|
||
<div class="reverse-visual">
|
||
<div><span>NORMAL</span><code>A B C D → C → w x y z</code><i class="long"></i><p>A 到 w 的路径很长</p></div>
|
||
<div><span>REVERSED</span><code>D C B A → C → w x y z</code><i class="short"></i><p>常见对齐的最小时间距离缩短</p></div>
|
||
</div>
|
||
<p>
|
||
这项技巧不增加理论容量，而是重排计算图，让相互对应的源词和目标词更近，优化更容易。
|
||
它预告了一个重要主题：信息路径长度本身就是架构属性。
|
||
</p>
|
||
|
||
<div class="bottleneck-callout">
|
||
<span>THE FIXED-VECTOR BOTTLENECK</span>
|
||
<p>
|
||
5 个词和 50 个词都要压进同样大小的 C。decoder 生成每一步时，看不到 encoder 的逐位置记忆，
|
||
只能相信 C 已经提前保存了未来每一步可能需要的一切。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="attention-bridge">
|
||
<p class="eyebrow"><span>16</span> SOFT ALIGNMENT</p>
|
||
<h2>Attention 最初是一座桥：每生成一步，重新读取一次源序列</h2>
|
||
<p>
|
||
Bahdanau、Cho 与 Bengio 明确把 fixed-length vector 称为基本瓶颈。
|
||
新做法不是只把最后 encoder state 交给 decoder，而是保留全部
|
||
\(h_1,\ldots,h_n\)，每个目标步都计算一组新的软对齐权重。
|
||
</p>
|
||
|
||
<div class="attention-equations">
|
||
<div><span>1 / SCORE</span><code>eₜᵢ = a(sₜ₋₁, hᵢ)</code><p>当前 decoder 状态与每个源状态匹配。</p></div>
|
||
<div><span>2 / NORMALIZE</span><code>αₜ = softmax(eₜ)</code><p>得到和为 1 的连续权重。</p></div>
|
||
<div><span>3 / READ</span><code>cₜ = Σᵢ αₜᵢhᵢ</code><p>每个目标步读出不同 context vector。</p></div>
|
||
</div>
|
||
|
||
<div class="alignment-table">
|
||
<div class="corner"><span>TARGET ↓ / SOURCE →</span></div>
|
||
{["the", "black", "cat", "sat"].map((token) => <div class="head">{token}</div>)}
|
||
<div class="target">黑色的</div><i style="--a:.04"></i><i style="--a:.76"></i><i style="--a:.14"></i><i style="--a:.06"></i>
|
||
<div class="target">猫</div><i style="--a:.03"></i><i style="--a:.09"></i><i style="--a:.82"></i><i style="--a:.06"></i>
|
||
<div class="target">坐下</div><i style="--a:.05"></i><i style="--a:.04"></i><i style="--a:.17"></i><i style="--a:.74"></i>
|
||
</div>
|
||
|
||
<div class="boundary-pair">
|
||
<article><span>BAHDANAU / ADDITIVE</span><h3>小网络打分</h3><p>query/state 与 key/encoder state 经投影、tanh 和向量打分；仍包在 RNN encoder–decoder 中。</p></article>
|
||
<article><span>TRANSFORMER / DOT-PRODUCT</span><h3>QKᵀ 矩阵打分</h3><p>scaled dot-product、多头、自注意力和位置表示属于下一章，公式不能倒写进 2014 论文。</p></article>
|
||
</div>
|
||
|
||
<h3>Attention 为什么最终能从桥变成主体</h3>
|
||
<ul class="plain-list">
|
||
<li>任意两个位置之间可以在一层内建立直接信息路径；</li>
|
||
<li>所有位置的打分和加权读取可以写成大矩阵运算；</li>
|
||
<li>不再要求训练沿时间步串行等待隐藏状态；</li>
|
||
<li>同一机制既能连接 encoder–decoder，也能让序列内部位置互相读取。</li>
|
||
</ul>
|
||
|
||
<div class="warning-note">
|
||
<b>注意力权重不是自动的因果解释</b>
|
||
<p>
|
||
它展示某个头、某层、某次前向中的信息混合系数；多个层、残差、Value 表示和非线性都会继续改变结果。
|
||
“看得多”不必然等于“对最终预测因果贡献最大”。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="economics">
|
||
<p class="eyebrow"><span>17</span> EVIDENCE & ECONOMICS</p>
|
||
<h2>一条技术路线能否成为主流，还要过词表、基准与硬件三道门</h2>
|
||
|
||
<div class="economics-grid">
|
||
<article>
|
||
<span>VOCABULARY TAX</span><h3>输出归一化</h3>
|
||
<p>全词表 Softmax 随 \(|V|\) 增长；hierarchical / sampled / adaptive 方法改变训练或输出结构。</p>
|
||
</article>
|
||
<article>
|
||
<span>SEQUENCE TAX</span><h3>时间依赖</h3>
|
||
<p>RNN 每步依赖前一步；卷积和 attention 更容易把训练摊成并行矩阵计算。</p>
|
||
</article>
|
||
<article>
|
||
<span>MEMORY TAX</span><h3>状态与激活</h3>
|
||
<p>固定状态省内存却压缩历史；保留逐位置表示提升检索，也增加存储和计算。</p>
|
||
</article>
|
||
<article>
|
||
<span>EVIDENCE TAX</span><h3>公平比较</h3>
|
||
<p>词表、预处理、数据量、ensemble 与硬件不同，单个 PPL 或 BLEU 不能脱离设置传播。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<h3>One Billion Word Benchmark 的价值与边界</h3>
|
||
<p>
|
||
Chelba 等 2013 提供接近十亿词的可重建 benchmark。在其设置下，unpruned Kneser–Ney 5-gram
|
||
baseline PPL 为 67.6；组合技术报告 35% PPL 降低（约 10% cross-entropy 降低），RNN 模型最佳。
|
||
这些数字不能与不同 tokenizer、不同测试集的现代 LLM PPL 直接比较。
|
||
</p>
|
||
|
||
<div class="ppl-nonlinear">
|
||
<div><span>BASELINE</span><b>PPL 67.6</b><em>H ≈ ln 67.6 = 4.21 nats</em></div>
|
||
<i>35% PPL ↓</i>
|
||
<div><span>COMBINATION</span><b>PPL ≈ 43.9</b><em>H ≈ ln 43.9 = 3.78 nats</em></div>
|
||
<p>35% 的 PPL 下降约对应 10% 的交叉熵下降；PPL 的指数尺度会放大视觉差异。</p>
|
||
</div>
|
||
|
||
<h3>为什么训练吞吐改变架构命运</h3>
|
||
<div class="parallel-compare">
|
||
<article>
|
||
<span>RNN</span>
|
||
<div><b>t1</b><i>→</i><b>t2</b><i>→</i><b>t3</b><i>→</i><b>t4</b></div>
|
||
<p>序列长度就是依赖链长度；GPU 很多单元要等前一步。</p>
|
||
</article>
|
||
<article>
|
||
<span>CONV / ATTENTION</span>
|
||
<div><b>t1</b><b>t2</b><b>t3</b><b>t4</b><strong>↓↓↓↓</strong><em>matrix kernels</em></div>
|
||
<p>训练时多个位置可批量计算，更符合大规模加速器。</p>
|
||
</article>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="modern">
|
||
<p class="eyebrow"><span>18</span> MODERN ECHO</p>
|
||
<h2>回到 K3 与 DeepSeek：主目标仍是 next-token，变化发生在“条件函数与辅助路径”</h2>
|
||
<p class="lede">
|
||
从 1951 的 next-letter predictability 到 2026 的原生多模态 MoE，
|
||
最稳妥的连续性不是“它们是同一个模型”，而是都把序列规律转成条件预测问题。
|
||
</p>
|
||
|
||
<div class="objective-stack">
|
||
<div class="main">
|
||
<span>MAIN OBJECTIVE</span>
|
||
<h3>Next-token NLL</h3>
|
||
<code>−log P(xₜ | x<ₜ)</code>
|
||
<p>定义因果主干学习什么分布。</p>
|
||
</div>
|
||
<i>+</i>
|
||
<div>
|
||
<span>AUXILIARY</span><h3>MTP</h3><p>额外预测未来 token，增密训练信号，并可成为 speculative draft 的桥。</p>
|
||
</div>
|
||
<i>+</i>
|
||
<div>
|
||
<span>CONDITION SPACE</span><h3>Multimodal</h3><p>视觉 token 进入同一序列/条件接口，不是 MTP 的一种。</p>
|
||
</div>
|
||
<i>+</i>
|
||
<div>
|
||
<span>POST-TRAINING</span><h3>SFT / RL</h3><p>改变行为分布与任务偏好，生成仍常沿因果 token 序列展开。</p>
|
||
</div>
|
||
</div>
|
||
|
||
<h3>DeepSeek：主 NTP 上叠一层顺序 MTP</h3>
|
||
<div class="mtp-diagram">
|
||
<div class="backbone"><span>MAIN MODEL</span><b>hₜ</b><i>→</i><strong>xₜ₊₁</strong><small>标准下一 token</small></div>
|
||
<div class="mtp"><span>MTP MODULE · D=1</span><b>h′ₜ</b><i>→</i><strong>xₜ₊₂</strong><small>额外未来 token</small></div>
|
||
</div>
|
||
<p>
|
||
DeepSeek-V3 报告的顺序 MTP module 保持 causal chain；预训练使用 \(D=1\)，
|
||
即标准下一 token 之外再预测一个额外 token。模块可在推理时丢弃，也可服务 speculative decoding。
|
||
V4 报告继承 V3 的 MTP strategy，并给出 MTP loss weight 从 0.3 调到 0.1 的阶段变化。
|
||
</p>
|
||
|
||
<h3>K3：统一视觉/文本 NTP 与 one MTP layer 同时成立</h3>
|
||
<div class="k3-token-stream">
|
||
<div><span>TEXT</span><b>请解释</b></div>
|
||
<div class="vision"><span>VISION</span><b>[image tokens]</b></div>
|
||
<div><span>TEXT</span><b>这张图</b></div>
|
||
<i>→</i>
|
||
<strong>同一因果 Token 流</strong>
|
||
<em>next-token objective</em>
|
||
</div>
|
||
<p>
|
||
K3 报告写明视觉与文本 token 交错，并在统一 next-token prediction objective 下训练；
|
||
配置表同时给出 160K vocabulary 与 one MTP layer。预训练后的 MTP layer
|
||
还能微调为 EAGLE-3 draft，在 target 冻结时学习贴近 target next-token distribution。
|
||
</p>
|
||
|
||
<div class="modern-contract">
|
||
<div class="head"><b>角色</b><b>DeepSeek-V3/V4</b><b>Kimi K3</b></div>
|
||
<div><span>主目标</span><p>因果 next-token NLL</p><p>统一视觉/文本 next-token objective</p></div>
|
||
<div><span>辅助预测</span><p>顺序 MTP，V3 预训练 D=1</p><p>one MTP layer</p></div>
|
||
<div><span>推理关系</span><p>可弃用或用于 speculative decoding</p><p>MTP → EAGLE-3 draft bridge</p></div>
|
||
<div><span>词表</span><p>128K byte-level BPE</p><p>160K vocabulary</p></div>
|
||
<div><span>不能说</span><p>“一次生成多 token 已取代自回归”</p><p>“多模态就是 MTP”</p></div>
|
||
</div>
|
||
|
||
<div class="deepseek-highlight">
|
||
<span>DEEPSEEK SPOTLIGHT / 为什么值得单独看</span>
|
||
<h3>它经常保留一个古老主目标，却在训练信号、稀疏容量和系统成本上重做合同</h3>
|
||
<p>
|
||
DeepSeek 的连续价值不只是某个组件名：DeepSeekMoE 改容量账、MLA 改缓存账、
|
||
MTP 改训练信号密度、FP8 与 DualPipe 改系统账。把这些层分开，才能看清“目标没变”与“训练体系大变”如何同时成立。
|
||
</p>
|
||
<a href="/deepseek/">进入 DeepSeek 专题 →</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="synthesis">
|
||
<p class="eyebrow"><span>19</span> SYNTHESIS</p>
|
||
<h2>把四次技术转弯亲手验证，再合成一条完整因果链</h2>
|
||
<LanguageModelHistoryLab />
|
||
|
||
<div class="causal-chain">
|
||
<article><span>01</span><h3>序列可分解</h3><p>链式法则把整段概率拆成逐步条件预测。</p></article>
|
||
<i>↓</i>
|
||
<article><span>02</span><h3>完整历史不可计数</h3><p>N-gram 用有限后缀换取可估计性。</p></article>
|
||
<i>↓</i>
|
||
<article><span>03</span><h3>组合仍极度稀疏</h3><p>折扣、回退与 continuation probability 处理未见事件。</p></article>
|
||
<i>↓</i>
|
||
<article><span>04</span><h3>精确符号不能柔性共享</h3><p>Embedding 让相似词在连续空间共享参数。</p></article>
|
||
<i>↓</i>
|
||
<article><span>05</span><h3>固定窗口看不远</h3><p>RNN 把变长前缀滚入固定状态。</p></article>
|
||
<i>↓</i>
|
||
<article><span>06</span><h3>重复传递让梯度衰减</h3><p>LSTM / GRU 用门控通路选择性保留与写入。</p></article>
|
||
<i>↓</i>
|
||
<article><span>07</span><h3>序列到序列压缩成单向量</h3><p>Attention 让每个输出步重新检索源状态。</p></article>
|
||
<i>↓</i>
|
||
<article class="final"><span>08</span><h3>桥梁成为主体</h3><p>Transformer 用并行自注意力重做上下文函数；下一章从这里开始。</p></article>
|
||
</div>
|
||
|
||
<h3>如果你现在能回答这十个问题，本章就真正读通了</h3>
|
||
<ol class="checkpoint-list">
|
||
<li><span>01</span><p>链式法则与 N-gram 的 Markov 假设有什么不同？</p></li>
|
||
<li><span>02</span><p>为什么一个未见 N-gram 会让整段最大似然概率归零？</p></li>
|
||
<li><span>03</span><p>Backoff 与 interpolation 的行为差别是什么？</p></li>
|
||
<li><span>04</span><p>Kneser–Ney 为什么不直接用普通 unigram 频率？</p></li>
|
||
<li><span>05</span><p>Bengio 2003 用连续表示解决了哪种“维数灾难”，又留下什么？</p></li>
|
||
<li><span>06</span><p>Word2Vec 为什么不是完整句子语言模型？</p></li>
|
||
<li><span>07</span><p>RNN 没有硬窗口，为什么仍不等于拥有无限可用记忆？</p></li>
|
||
<li><span>08</span><p>LSTM 的近恒等 cell path 如何改变梯度，仍未解决什么系统瓶颈？</p></li>
|
||
<li><span>09</span><p>Seq2Seq 的固定向量为何成为瓶颈，Attention 怎样按步骤解除它？</p></li>
|
||
<li><span>10</span><p>K3 的统一多模态 NTP、MTP layer 与 EAGLE-3 draft 分别扮演什么角色？</p></li>
|
||
</ol>
|
||
|
||
<div class="next-chapter">
|
||
<div>
|
||
<span>NEXT / CHAPTER 02</span>
|
||
<h3>Attention 已经出现；下一步是让它成为整个网络的主干</h3>
|
||
<p>继续拆 Q / K / V、scaled dot-product、multi-head、causal mask、位置与 Transformer Block。</p>
|
||
</div>
|
||
<a href="/foundations/">进入注意力与 Transformer →</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section papers-section" id="papers">
|
||
<p class="eyebrow"><span>↳</span> PRIMARY PAPER CHAIN</p>
|
||
<h2>33 个正式节点：主干负责走通，地图负责避免伪单线史</h2>
|
||
<p>
|
||
每条均链接原论文、出版社/学会页面或作者预印本。“主干”建议按顺序精读；
|
||
“地图”用于理解同时代旁支与系统约束；当代报告把目标历史接回 K3 / DeepSeek。
|
||
</p>
|
||
|
||
<div class="paper-legend">
|
||
<span><i class="core"></i>主干 · 建议精读</span>
|
||
<span><i class="map"></i>地图 · 建议查阅</span>
|
||
<span><i class="now"></i>当代 · 报告映射</span>
|
||
</div>
|
||
|
||
<div class="paper-chain">
|
||
{keyPapers.map((paper, index) => (
|
||
<a href={paper.url} target="_blank" rel="noreferrer" class:list={[paper.tier === "主干" ? "core" : paper.tier === "当代" ? "now" : "map"]}>
|
||
<span class="paper-index">{String(index + 1).padStart(2, "0")}</span>
|
||
<time>{paper.year}</time>
|
||
<div><h3>{paper.title}</h3><p>{paper.who}</p></div>
|
||
<em>{paper.role}</em>
|
||
<b>{paper.tier} ↗</b>
|
||
</a>
|
||
))}
|
||
</div>
|
||
|
||
<div class="research-ledger-link">
|
||
<span>REPRODUCIBLE RESEARCH</span>
|
||
<p>
|
||
逐篇证据边界、公式合同、现代报告本地行号和 Grok 未核验队列均保存在开源仓库
|
||
<code>research/LANGUAGE_MODEL_HISTORY_RESEARCH.md</code>。
|
||
</p>
|
||
<a href="https://git.k1412.top/wuyang/llm-atlas" target="_blank" rel="noreferrer">查看公开研究账本 ↗</a>
|
||
</div>
|
||
</section>
|
||
</article>
|
||
</div>
|
||
|
||
<style>
|
||
.lm-hero {
|
||
background:
|
||
radial-gradient(circle at 78% 18%, color-mix(in srgb, var(--copper) 12%, transparent), transparent 27%),
|
||
var(--paper);
|
||
}
|
||
|
||
.hero-question {
|
||
max-width: 760px;
|
||
margin: 24px 0 0;
|
||
padding: 14px 0 14px 20px;
|
||
border-left: 3px solid var(--copper);
|
||
color: var(--copper);
|
||
font-family: var(--serif);
|
||
font-size: clamp(1.15rem, 2.2vw, 1.6rem);
|
||
font-style: italic;
|
||
}
|
||
|
||
.evidence-note {
|
||
display: grid;
|
||
gap: 8px;
|
||
}
|
||
|
||
.evidence-note span {
|
||
display: flex;
|
||
gap: 8px;
|
||
align-items: center;
|
||
color: var(--muted);
|
||
font-size: 0.61rem;
|
||
}
|
||
|
||
.evidence-note i,
|
||
.paper-legend i {
|
||
display: inline-block;
|
||
width: 8px;
|
||
height: 8px;
|
||
border-radius: 50%;
|
||
}
|
||
|
||
.evidence-note .verified,
|
||
.paper-legend .core { background: var(--copper); }
|
||
.evidence-note .explain,
|
||
.paper-legend .map { border: 1px solid var(--muted); }
|
||
.evidence-note .inference,
|
||
.paper-legend .now { background: var(--ink); }
|
||
|
||
.ledger-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||
max-width: 1080px;
|
||
margin: 32px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.ledger-grid article {
|
||
min-height: 210px;
|
||
padding: 22px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.ledger-grid span,
|
||
.reading-contract span,
|
||
.prediction-demo > p,
|
||
.source-boundary > span,
|
||
.concept-split > article > span,
|
||
.three-choices span,
|
||
.metric-ladder span,
|
||
.unit-comparison > article > span,
|
||
.tokenizer-cost span,
|
||
.modern-note > span,
|
||
.ngram-window > div > span,
|
||
.tradeoff-axis span,
|
||
.old-is-smart > span,
|
||
.smoothing-pipeline span,
|
||
.backoff-tree span,
|
||
.continuation-compare > article > span,
|
||
.evidence-row span,
|
||
.sharing-staircase span,
|
||
.representation-boundary span,
|
||
.nplm-diagram span,
|
||
.bottleneck-grid span,
|
||
.word2vec-compare > article > span,
|
||
.negative-sampling span,
|
||
.static-limit > span,
|
||
.fixed-window-vs-state span,
|
||
.gradient-regimes span,
|
||
.lstm-diagram span,
|
||
.history-correction > span,
|
||
.still-open span,
|
||
.claim-contract span,
|
||
.softmax-routes span,
|
||
.seq2seq-diagram span,
|
||
.reverse-visual span,
|
||
.bottleneck-callout > span,
|
||
.attention-equations span,
|
||
.boundary-pair span,
|
||
.economics-grid span,
|
||
.ppl-nonlinear span,
|
||
.parallel-compare > article > span,
|
||
.objective-stack span,
|
||
.mtp-diagram span,
|
||
.k3-token-stream span,
|
||
.deepseek-highlight > span,
|
||
.causal-chain span,
|
||
.next-chapter span,
|
||
.research-ledger-link > span {
|
||
color: var(--copper);
|
||
font: 0.61rem/1.45 var(--mono);
|
||
letter-spacing: 0.08em;
|
||
}
|
||
|
||
.ledger-grid h3 {
|
||
margin: 16px 0 9px;
|
||
font-size: 1.1rem;
|
||
}
|
||
|
||
.ledger-grid b {
|
||
display: block;
|
||
min-height: 34px;
|
||
color: var(--muted);
|
||
font-size: 0.72rem;
|
||
}
|
||
|
||
.ledger-grid p {
|
||
margin-top: 16px;
|
||
color: var(--ink);
|
||
font: 0.69rem/1.5 var(--mono);
|
||
}
|
||
|
||
.ledger-grid small {
|
||
display: block;
|
||
margin-top: 14px;
|
||
color: var(--muted-light);
|
||
font-size: 0.63rem;
|
||
line-height: 1.5;
|
||
}
|
||
|
||
.reading-contract {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 980px;
|
||
margin: 36px 0 52px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.reading-contract > div {
|
||
padding: 22px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.reading-contract > div:last-child {
|
||
border-right: 0;
|
||
}
|
||
|
||
.reading-contract p {
|
||
margin-top: 10px;
|
||
color: var(--muted);
|
||
font-size: 0.7rem;
|
||
line-height: 1.7;
|
||
}
|
||
|
||
.era-map {
|
||
max-width: 960px;
|
||
margin: 28px 0 14px;
|
||
}
|
||
|
||
.era-map > div {
|
||
display: grid;
|
||
grid-template-columns: 110px 36px 1fr;
|
||
min-height: 98px;
|
||
}
|
||
|
||
.era-map > div > span {
|
||
padding-top: 4px;
|
||
color: var(--copper);
|
||
font: 0.65rem/1.4 var(--mono);
|
||
}
|
||
|
||
.era-map > div > i {
|
||
position: relative;
|
||
display: grid;
|
||
width: 26px;
|
||
height: 26px;
|
||
place-items: center;
|
||
border: 1px solid var(--line-strong);
|
||
border-radius: 50%;
|
||
color: var(--muted);
|
||
font: 0.55rem/1 var(--mono);
|
||
}
|
||
|
||
.era-map > div:not(:last-child) > i::after {
|
||
position: absolute;
|
||
top: 26px;
|
||
left: 12px;
|
||
width: 1px;
|
||
height: 72px;
|
||
background: var(--line);
|
||
content: "";
|
||
}
|
||
|
||
.era-map article {
|
||
padding: 0 0 22px 20px;
|
||
}
|
||
|
||
.era-map h4 {
|
||
margin-bottom: 7px;
|
||
font-size: 1rem;
|
||
}
|
||
|
||
.era-map p,
|
||
.figure-note {
|
||
color: var(--muted);
|
||
font-size: 0.72rem;
|
||
line-height: 1.7;
|
||
}
|
||
|
||
.figure-note {
|
||
max-width: 900px;
|
||
padding: 14px 0;
|
||
border-top: 1px solid var(--line);
|
||
}
|
||
|
||
.figure-note b {
|
||
margin-right: 8px;
|
||
color: var(--copper);
|
||
font: 0.62rem/1 var(--mono);
|
||
}
|
||
|
||
.prediction-demo {
|
||
display: grid;
|
||
grid-template-columns: 220px minmax(320px, 1fr);
|
||
max-width: 900px;
|
||
margin: 34px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.context-strip {
|
||
display: grid;
|
||
align-content: center;
|
||
gap: 15px;
|
||
padding: 28px;
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.context-strip span {
|
||
color: color-mix(in srgb, var(--paper-raised) 58%, transparent);
|
||
font: 0.6rem/1 var(--mono);
|
||
}
|
||
|
||
.context-strip b {
|
||
font-size: 1.28rem;
|
||
}
|
||
|
||
.candidate-list {
|
||
display: grid;
|
||
gap: 13px;
|
||
padding: 26px;
|
||
}
|
||
|
||
.candidate-list > div {
|
||
display: grid;
|
||
grid-template-columns: 42px 1fr 42px;
|
||
gap: 12px;
|
||
align-items: center;
|
||
}
|
||
|
||
.candidate-list span {
|
||
font-size: 0.78rem;
|
||
}
|
||
|
||
.candidate-list i {
|
||
height: 8px;
|
||
background: var(--copper);
|
||
}
|
||
|
||
.candidate-list b {
|
||
color: var(--muted);
|
||
font: 0.66rem/1 var(--mono);
|
||
}
|
||
|
||
.prediction-demo > p {
|
||
grid-column: 1 / -1;
|
||
padding: 11px 16px;
|
||
border-top: 1px solid var(--line);
|
||
color: var(--muted);
|
||
letter-spacing: 0;
|
||
}
|
||
|
||
.source-boundary {
|
||
display: grid;
|
||
grid-template-columns: 110px 1fr auto;
|
||
gap: 22px;
|
||
align-items: center;
|
||
max-width: 980px;
|
||
margin: 32px 0;
|
||
padding: 20px;
|
||
border: 1px solid var(--line);
|
||
border-left: 4px solid var(--copper);
|
||
}
|
||
|
||
.source-boundary h4 {
|
||
margin-bottom: 5px;
|
||
font-size: 0.92rem;
|
||
}
|
||
|
||
.source-boundary p {
|
||
color: var(--muted);
|
||
font-size: 0.68rem;
|
||
line-height: 1.55;
|
||
}
|
||
|
||
.source-boundary a {
|
||
color: var(--copper);
|
||
font: 0.65rem/1 var(--mono);
|
||
}
|
||
|
||
.concept-split,
|
||
.three-choices,
|
||
.metric-ladder,
|
||
.bottleneck-grid,
|
||
.still-open,
|
||
.claim-contract,
|
||
.boundary-pair,
|
||
.economics-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||
max-width: 1020px;
|
||
margin: 32px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.concept-split article,
|
||
.three-choices > div,
|
||
.metric-ladder article,
|
||
.bottleneck-grid article,
|
||
.still-open > div,
|
||
.claim-contract article,
|
||
.boundary-pair article,
|
||
.economics-grid article {
|
||
padding: 24px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.concept-split h3,
|
||
.three-choices h4,
|
||
.metric-ladder h3,
|
||
.bottleneck-grid h3,
|
||
.still-open b,
|
||
.claim-contract h3,
|
||
.boundary-pair h3,
|
||
.economics-grid h3 {
|
||
margin: 14px 0 9px;
|
||
font-size: 1.05rem;
|
||
}
|
||
|
||
.concept-split p,
|
||
.three-choices p,
|
||
.metric-ladder p,
|
||
.bottleneck-grid p,
|
||
.still-open p,
|
||
.claim-contract p,
|
||
.boundary-pair p,
|
||
.economics-grid p {
|
||
color: var(--muted);
|
||
font-size: 0.7rem;
|
||
line-height: 1.65;
|
||
}
|
||
|
||
.formula.large {
|
||
font-size: clamp(1rem, 2.4vw, 1.35rem);
|
||
}
|
||
|
||
.formula.compact {
|
||
margin: 15px 0 9px;
|
||
padding: 14px;
|
||
font-size: 0.72rem;
|
||
}
|
||
|
||
.chain-visual {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1.3fr auto 1.55fr auto 1.8fr;
|
||
max-width: 1040px;
|
||
margin: 34px 0;
|
||
align-items: stretch;
|
||
}
|
||
|
||
.chain-visual > div {
|
||
display: grid;
|
||
gap: 9px;
|
||
padding: 18px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.chain-visual > div > span {
|
||
color: var(--copper);
|
||
font: 0.58rem/1 var(--mono);
|
||
}
|
||
|
||
.chain-visual > div > b {
|
||
font-size: 1.15rem;
|
||
}
|
||
|
||
.chain-visual > div > em {
|
||
color: var(--muted);
|
||
font: 0.6rem/1.4 var(--mono);
|
||
}
|
||
|
||
.chain-visual > i {
|
||
align-self: center;
|
||
padding: 0 8px;
|
||
color: var(--copper);
|
||
font: 0.8rem/1 var(--mono);
|
||
}
|
||
|
||
.worked-example {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 860px;
|
||
margin: 26px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.worked-example > div {
|
||
display: grid;
|
||
gap: 9px;
|
||
padding: 24px;
|
||
}
|
||
|
||
.worked-example > div + div {
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.worked-example span {
|
||
color: var(--copper);
|
||
font: 0.62rem/1 var(--mono);
|
||
}
|
||
|
||
.worked-example p,
|
||
.worked-example b,
|
||
.worked-example em {
|
||
font-size: 0.72rem;
|
||
}
|
||
|
||
.worked-example p { color: var(--muted); }
|
||
.worked-example em { color: var(--copper); }
|
||
|
||
.unit-comparison {
|
||
display: grid;
|
||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||
max-width: 1080px;
|
||
margin: 32px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.unit-comparison article {
|
||
min-height: 250px;
|
||
padding: 24px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.unit-comparison article:last-child {
|
||
border-right: 0;
|
||
}
|
||
|
||
.unit-comparison article.accent {
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.unit-comparison article.accent p {
|
||
color: color-mix(in srgb, var(--paper-raised) 64%, transparent);
|
||
}
|
||
|
||
.unit-comparison h3 {
|
||
margin: 18px 0 12px;
|
||
font-size: 1.2rem;
|
||
}
|
||
|
||
.unit-comparison p {
|
||
min-height: 92px;
|
||
color: var(--muted);
|
||
font-size: 0.7rem;
|
||
line-height: 1.65;
|
||
}
|
||
|
||
.unit-comparison code {
|
||
display: block;
|
||
margin-top: 15px;
|
||
color: var(--copper);
|
||
font-size: 0.7rem;
|
||
}
|
||
|
||
.bpe-steps {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1fr auto 1fr;
|
||
max-width: 900px;
|
||
margin: 28px 0;
|
||
align-items: center;
|
||
}
|
||
|
||
.bpe-steps > div {
|
||
display: grid;
|
||
gap: 9px;
|
||
min-height: 132px;
|
||
padding: 20px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.bpe-steps span {
|
||
color: var(--copper);
|
||
font: 0.6rem/1 var(--mono);
|
||
}
|
||
|
||
.bpe-steps code {
|
||
font-size: 0.83rem;
|
||
}
|
||
|
||
.bpe-steps p {
|
||
color: var(--muted);
|
||
font-size: 0.67rem;
|
||
}
|
||
|
||
.bpe-steps > i {
|
||
padding: 0 12px;
|
||
color: var(--copper);
|
||
}
|
||
|
||
.tokenizer-cost {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1fr;
|
||
max-width: 900px;
|
||
margin: 34px 0;
|
||
align-items: center;
|
||
border-top: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.tokenizer-cost > div {
|
||
padding: 22px;
|
||
}
|
||
|
||
.tokenizer-cost b {
|
||
display: block;
|
||
margin: 8px 0;
|
||
font-size: 1rem;
|
||
}
|
||
|
||
.tokenizer-cost p {
|
||
color: var(--muted);
|
||
font-size: 0.68rem;
|
||
}
|
||
|
||
.tokenizer-cost > i {
|
||
color: var(--copper);
|
||
font-size: 1.4rem;
|
||
}
|
||
|
||
.modern-note,
|
||
.old-is-smart,
|
||
.bottleneck-callout {
|
||
max-width: 960px;
|
||
margin: 30px 0;
|
||
padding: 24px 28px;
|
||
border-left: 4px solid var(--copper);
|
||
background: var(--paper-deep);
|
||
}
|
||
|
||
.modern-note p,
|
||
.old-is-smart p,
|
||
.bottleneck-callout p {
|
||
margin-top: 9px;
|
||
color: var(--muted);
|
||
font-size: 0.73rem;
|
||
line-height: 1.75;
|
||
}
|
||
|
||
.ngram-window {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 960px;
|
||
margin: 30px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.ngram-window > div {
|
||
padding: 22px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.ngram-window > div:last-child {
|
||
border-right: 0;
|
||
}
|
||
|
||
.ngram-window > div > div {
|
||
display: flex;
|
||
gap: 6px;
|
||
margin: 18px 0;
|
||
}
|
||
|
||
.ngram-window i,
|
||
.ngram-window em,
|
||
.ngram-window b {
|
||
padding: 6px 7px;
|
||
border: 1px solid var(--line);
|
||
color: var(--muted-light);
|
||
font: 0.66rem/1 var(--mono);
|
||
}
|
||
|
||
.ngram-window em {
|
||
border-color: var(--copper);
|
||
color: var(--copper);
|
||
}
|
||
|
||
.ngram-window b {
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.ngram-window p {
|
||
color: var(--muted);
|
||
font-size: 0.66rem;
|
||
line-height: 1.55;
|
||
}
|
||
|
||
.tradeoff-axis {
|
||
display: grid;
|
||
grid-template-columns: 230px 1fr 230px;
|
||
max-width: 940px;
|
||
margin: 32px 0;
|
||
align-items: center;
|
||
}
|
||
|
||
.tradeoff-axis > div:not(.axis-line) {
|
||
padding: 18px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.tradeoff-axis b {
|
||
display: block;
|
||
margin: 8px 0;
|
||
}
|
||
|
||
.tradeoff-axis p {
|
||
color: var(--muted);
|
||
font-size: 0.66rem;
|
||
}
|
||
|
||
.axis-line {
|
||
height: 1px;
|
||
background: var(--line-strong);
|
||
}
|
||
|
||
.axis-line i {
|
||
display: block;
|
||
width: 50%;
|
||
height: 3px;
|
||
margin-top: -1px;
|
||
background: var(--copper);
|
||
}
|
||
|
||
.zero-chain {
|
||
display: grid;
|
||
grid-template-columns: repeat(9, auto);
|
||
max-width: 1000px;
|
||
margin: 32px 0;
|
||
align-items: center;
|
||
justify-content: start;
|
||
}
|
||
|
||
.zero-chain > div {
|
||
display: grid;
|
||
gap: 7px;
|
||
min-width: 110px;
|
||
padding: 18px 14px;
|
||
border: 1px solid var(--line);
|
||
text-align: center;
|
||
}
|
||
|
||
.zero-chain > div.zero {
|
||
border-color: #a13b2d;
|
||
background: color-mix(in srgb, #a13b2d 8%, var(--paper));
|
||
}
|
||
|
||
.zero-chain > div.zero b,
|
||
.zero-chain > strong {
|
||
color: #a13b2d;
|
||
}
|
||
|
||
.zero-chain b {
|
||
font: 1.15rem/1 var(--mono);
|
||
}
|
||
|
||
.zero-chain span {
|
||
color: var(--muted);
|
||
font: 0.55rem/1.4 var(--mono);
|
||
}
|
||
|
||
.zero-chain > i {
|
||
padding: 0 8px;
|
||
color: var(--copper);
|
||
}
|
||
|
||
.zero-chain > strong {
|
||
padding-left: 12px;
|
||
font: 2rem/1 var(--mono);
|
||
}
|
||
|
||
.smoothing-pipeline {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1fr auto 1fr;
|
||
max-width: 960px;
|
||
margin: 28px 0;
|
||
align-items: center;
|
||
}
|
||
|
||
.smoothing-pipeline article {
|
||
min-height: 170px;
|
||
padding: 22px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.smoothing-pipeline h4 {
|
||
margin: 17px 0 10px;
|
||
font-size: 1.08rem;
|
||
}
|
||
|
||
.smoothing-pipeline p {
|
||
color: var(--muted);
|
||
font-size: 0.69rem;
|
||
line-height: 1.6;
|
||
}
|
||
|
||
.smoothing-pipeline > i {
|
||
padding: 0 12px;
|
||
color: var(--copper);
|
||
}
|
||
|
||
.backoff-tree {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 850px;
|
||
margin: 28px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.backoff-tree > div {
|
||
padding: 22px;
|
||
}
|
||
|
||
.backoff-tree > div:first-child {
|
||
grid-column: 1 / -1;
|
||
border-bottom: 1px solid var(--line);
|
||
text-align: center;
|
||
}
|
||
|
||
.backoff-tree .branch + .branch {
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.backoff-tree b {
|
||
display: block;
|
||
margin: 9px 0;
|
||
}
|
||
|
||
.backoff-tree em,
|
||
.backoff-tree p {
|
||
color: var(--muted);
|
||
font-size: 0.66rem;
|
||
}
|
||
|
||
.myth-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 900px;
|
||
margin: 28px 0;
|
||
border-top: 1px solid var(--line);
|
||
}
|
||
|
||
.myth-grid > div {
|
||
padding: 20px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.myth-grid span {
|
||
color: var(--ink);
|
||
font-weight: 700;
|
||
}
|
||
|
||
.myth-grid p {
|
||
margin-top: 8px;
|
||
color: var(--muted);
|
||
font-size: 0.67rem;
|
||
}
|
||
|
||
.continuation-compare {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 980px;
|
||
margin: 32px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.continuation-compare article {
|
||
min-height: 360px;
|
||
padding: 28px;
|
||
}
|
||
|
||
.continuation-compare article + article {
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.continuation-compare article.accent {
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.continuation-compare h3 {
|
||
margin: 16px 0 28px;
|
||
font-size: 1.2rem;
|
||
}
|
||
|
||
.continuation-compare article > p {
|
||
margin-top: 28px;
|
||
color: var(--muted);
|
||
font-size: 0.7rem;
|
||
line-height: 1.6;
|
||
}
|
||
|
||
.continuation-compare article.accent > p {
|
||
color: color-mix(in srgb, var(--paper-raised) 62%, transparent);
|
||
}
|
||
|
||
.count-bars {
|
||
display: grid;
|
||
gap: 18px;
|
||
}
|
||
|
||
.count-bars > div {
|
||
display: grid;
|
||
grid-template-columns: 85px 1fr 42px;
|
||
gap: 10px;
|
||
align-items: center;
|
||
}
|
||
|
||
.count-bars b,
|
||
.count-bars em {
|
||
font-size: 0.65rem;
|
||
}
|
||
|
||
.count-bars em { color: var(--muted); }
|
||
.count-bars i { height: 9px; background: var(--copper); }
|
||
|
||
.context-cloud {
|
||
display: grid;
|
||
gap: 18px;
|
||
}
|
||
|
||
.context-cloud > div {
|
||
display: grid;
|
||
gap: 8px;
|
||
padding: 14px;
|
||
border: 1px solid color-mix(in srgb, var(--paper-raised) 25%, transparent);
|
||
}
|
||
|
||
.context-cloud code {
|
||
color: var(--copper);
|
||
font-size: 0.67rem;
|
||
}
|
||
|
||
.evidence-row {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 1000px;
|
||
margin: 34px 0;
|
||
}
|
||
|
||
.evidence-row a {
|
||
display: grid;
|
||
gap: 10px;
|
||
padding: 21px;
|
||
border: 1px solid var(--line);
|
||
color: var(--ink);
|
||
}
|
||
|
||
.evidence-row a + a {
|
||
border-left: 0;
|
||
}
|
||
|
||
.evidence-row em {
|
||
color: var(--muted);
|
||
font-size: 0.65rem;
|
||
}
|
||
|
||
.sharing-staircase {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1fr auto 1.2fr;
|
||
max-width: 1020px;
|
||
margin: 34px 0;
|
||
align-items: stretch;
|
||
}
|
||
|
||
.sharing-staircase article {
|
||
padding: 22px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.sharing-staircase article.accent {
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.sharing-staircase h3 {
|
||
margin: 16px 0 9px;
|
||
}
|
||
|
||
.sharing-staircase p {
|
||
min-height: 55px;
|
||
color: var(--muted);
|
||
font-size: 0.68rem;
|
||
}
|
||
|
||
.sharing-staircase article.accent p {
|
||
color: color-mix(in srgb, var(--paper-raised) 65%, transparent);
|
||
}
|
||
|
||
.sharing-staircase code {
|
||
display: block;
|
||
margin-top: 16px;
|
||
color: var(--copper);
|
||
font-size: 0.6rem;
|
||
line-height: 1.5;
|
||
}
|
||
|
||
.sharing-staircase > i {
|
||
align-self: center;
|
||
padding: 0 10px;
|
||
color: var(--copper);
|
||
}
|
||
|
||
.representation-boundary {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 980px;
|
||
margin: 32px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.representation-boundary > div {
|
||
padding: 22px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.representation-boundary > div:last-child { border-right: 0; }
|
||
.representation-boundary b { display: block; margin: 12px 0; }
|
||
.representation-boundary p { color: var(--muted); font-size: 0.68rem; line-height: 1.6; }
|
||
|
||
.nplm-diagram {
|
||
display: grid;
|
||
grid-template-columns: 1.3fr auto 1fr auto .8fr auto .9fr;
|
||
max-width: 1080px;
|
||
margin: 36px 0;
|
||
align-items: center;
|
||
padding: 26px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.nplm-diagram > i {
|
||
padding: 0 10px;
|
||
color: var(--copper);
|
||
font: 0.62rem/1 var(--mono);
|
||
}
|
||
|
||
.nplm-inputs,
|
||
.nplm-embed,
|
||
.nplm-hidden,
|
||
.nplm-output {
|
||
display: grid;
|
||
gap: 10px;
|
||
padding: 18px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.nplm-inputs > div {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
gap: 10px;
|
||
padding: 7px;
|
||
background: var(--paper-deep);
|
||
}
|
||
|
||
.nplm-inputs b { font-size: 0.7rem; }
|
||
.nplm-inputs em,
|
||
.nplm-diagram small { color: var(--muted); font-size: 0.58rem; }
|
||
.nplm-diagram code { color: var(--copper); font-size: 0.68rem; }
|
||
|
||
.nplm-output {
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.numbered-explanation {
|
||
max-width: 900px;
|
||
margin: 25px 0;
|
||
padding: 0;
|
||
border-top: 1px solid var(--line);
|
||
list-style: none;
|
||
}
|
||
|
||
.numbered-explanation li {
|
||
display: grid;
|
||
grid-template-columns: 42px 1fr;
|
||
gap: 15px;
|
||
padding: 17px 0;
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.numbered-explanation span {
|
||
color: var(--copper);
|
||
font: 0.65rem/1.5 var(--mono);
|
||
}
|
||
|
||
.numbered-explanation p {
|
||
font-size: 0.74rem;
|
||
line-height: 1.65;
|
||
}
|
||
|
||
.word2vec-compare {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 980px;
|
||
margin: 32px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.word2vec-compare article {
|
||
padding: 26px;
|
||
}
|
||
|
||
.word2vec-compare article + article { border-left: 1px solid var(--line); }
|
||
.word2vec-compare h3 { margin: 20px 0 8px; }
|
||
.word2vec-compare p { color: var(--muted); font-size: 0.7rem; }
|
||
|
||
.window-diagram {
|
||
display: flex;
|
||
gap: 6px;
|
||
margin-top: 20px;
|
||
}
|
||
|
||
.window-diagram i,
|
||
.window-diagram b {
|
||
padding: 8px 7px;
|
||
border: 1px solid var(--line);
|
||
font: 0.65rem/1 var(--mono);
|
||
}
|
||
|
||
.window-diagram b {
|
||
border-color: var(--copper);
|
||
background: var(--copper);
|
||
color: white;
|
||
}
|
||
|
||
.negative-sampling {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 900px;
|
||
margin: 25px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.negative-sampling > div {
|
||
display: grid;
|
||
gap: 9px;
|
||
padding: 20px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.negative-sampling > div:last-child { border-right: 0; }
|
||
.negative-sampling .positive { background: var(--ink); color: var(--paper-raised); }
|
||
.negative-sampling b { font-size: 0.85rem; }
|
||
.negative-sampling em { color: var(--muted); font-size: 0.63rem; }
|
||
|
||
.static-limit {
|
||
display: grid;
|
||
grid-template-columns: 130px 1fr 1fr;
|
||
max-width: 960px;
|
||
margin: 32px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.static-limit > span,
|
||
.static-limit > div {
|
||
padding: 20px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.static-limit > div {
|
||
display: grid;
|
||
gap: 8px;
|
||
}
|
||
|
||
.static-limit code { color: var(--muted); font-size: 0.64rem; }
|
||
.static-limit > p { grid-column: 1 / -1; padding: 13px 20px; border-top: 1px solid var(--line); color: var(--muted); font-size: 0.67rem; }
|
||
|
||
.rnn-unroll {
|
||
display: grid;
|
||
grid-template-columns: repeat(7, auto);
|
||
max-width: 900px;
|
||
margin: 34px 0;
|
||
align-items: end;
|
||
justify-content: start;
|
||
}
|
||
|
||
.rnn-unroll > div {
|
||
display: grid;
|
||
grid-template-columns: 1fr;
|
||
gap: 7px;
|
||
justify-items: center;
|
||
min-width: 120px;
|
||
padding: 15px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.rnn-unroll span { color: var(--muted); font: 0.58rem/1 var(--mono); }
|
||
.rnn-unroll i { color: var(--copper); }
|
||
.rnn-unroll strong { padding: 7px 12px; background: var(--ink); color: var(--paper-raised); font: 0.7rem/1 var(--mono); }
|
||
.rnn-unroll > em { padding: 0 10px 22px; color: var(--copper); }
|
||
|
||
.fixed-window-vs-state {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 900px;
|
||
margin: 28px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.fixed-window-vs-state article { padding: 24px; }
|
||
.fixed-window-vs-state article + article { border-left: 1px solid var(--line); }
|
||
.fixed-window-vs-state h3 { margin: 15px 0 9px; }
|
||
.fixed-window-vs-state p { color: var(--muted); font-size: 0.68rem; }
|
||
.fixed-window-vs-state code { display: block; margin-top: 16px; color: var(--copper); font-size: 0.68rem; }
|
||
|
||
.gradient-regimes {
|
||
grid-template-columns: repeat(3, 1fr);
|
||
}
|
||
|
||
.decay-bars {
|
||
display: grid;
|
||
gap: 7px;
|
||
margin: 18px 0;
|
||
}
|
||
|
||
.decay-bars i {
|
||
display: block;
|
||
height: 7px;
|
||
background: var(--copper);
|
||
}
|
||
|
||
.decay-bars.stable i { width: 75%; }
|
||
.decay-bars.explode i:nth-child(1) { width: 18%; }
|
||
.decay-bars.explode i:nth-child(2) { width: 38%; }
|
||
.decay-bars.explode i:nth-child(3) { width: 68%; }
|
||
.decay-bars.explode i:nth-child(4) { width: 100%; }
|
||
|
||
.distance-story {
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
padding: 24px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.distance-story > span { font-family: var(--serif); font-size: 1.1rem; }
|
||
.distance-story > div { display: grid; grid-template-columns: auto repeat(7, 1fr) auto; gap: 5px; align-items: center; margin: 24px 0 12px; }
|
||
.distance-story i { height: 1px; background: var(--line-strong); }
|
||
.distance-story b,
|
||
.distance-story strong { padding: 8px 10px; background: var(--ink); color: var(--paper-raised); font-size: 0.65rem; }
|
||
.distance-story p { color: var(--muted); font-size: 0.68rem; }
|
||
|
||
.lstm-diagram {
|
||
max-width: 1000px;
|
||
margin: 34px 0;
|
||
padding: 28px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.cell-highway {
|
||
display: grid;
|
||
grid-template-columns: auto 1fr auto 100px auto 100px auto;
|
||
align-items: center;
|
||
color: var(--copper);
|
||
}
|
||
|
||
.cell-highway i { overflow: hidden; white-space: nowrap; }
|
||
.cell-highway b,
|
||
.cell-highway strong { display: grid; width: 44px; height: 44px; place-items: center; border: 1px solid var(--copper); border-radius: 50%; font: 0.68rem/1 var(--mono); }
|
||
|
||
.gate-row {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
gap: 14px;
|
||
margin-top: 32px;
|
||
}
|
||
|
||
.gate-row > div { padding: 18px; background: var(--paper-deep); }
|
||
.gate-row b { display: block; margin: 10px 0; font: 0.9rem/1 var(--mono); }
|
||
.gate-row p { color: var(--muted); font-size: 0.65rem; }
|
||
|
||
.history-correction {
|
||
display: grid;
|
||
grid-template-columns: 160px 1fr;
|
||
max-width: 920px;
|
||
margin: 32px 0;
|
||
padding: 22px;
|
||
border: 1px solid var(--line);
|
||
border-left: 4px solid var(--copper);
|
||
}
|
||
|
||
.history-correction h3 { margin-bottom: 8px; }
|
||
.history-correction p { color: var(--muted); font-size: 0.7rem; line-height: 1.65; }
|
||
|
||
.softmax-routes {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 980px;
|
||
margin: 30px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.softmax-routes article {
|
||
min-height: 260px;
|
||
padding: 22px;
|
||
border-right: 1px solid var(--line);
|
||
}
|
||
|
||
.softmax-routes article:last-child { border-right: 0; }
|
||
.softmax-routes p { margin-top: 18px; color: var(--muted); font-size: 0.67rem; }
|
||
|
||
.vocab-wall {
|
||
display: grid;
|
||
grid-template-columns: repeat(9, 1fr);
|
||
gap: 4px;
|
||
margin-top: 25px;
|
||
}
|
||
|
||
.vocab-wall i { aspect-ratio: 1; background: var(--line-strong); }
|
||
.vocab-wall i:nth-child(5n) { background: var(--copper); }
|
||
|
||
.tree-mini {
|
||
position: relative;
|
||
height: 120px;
|
||
margin-top: 20px;
|
||
background: linear-gradient(60deg, transparent 48%, var(--line-strong) 49% 51%, transparent 52%) 28% 15%/42% 55% no-repeat,
|
||
linear-gradient(-60deg, transparent 48%, var(--line-strong) 49% 51%, transparent 52%) 72% 15%/42% 55% no-repeat;
|
||
}
|
||
|
||
.tree-mini b,
|
||
.tree-mini i,
|
||
.tree-mini em { position: absolute; width: 10px; height: 10px; border-radius: 50%; background: var(--copper); }
|
||
.tree-mini b { left: 50%; top: 5px; }
|
||
.tree-mini i:nth-of-type(1) { left: 29%; top: 58px; }
|
||
.tree-mini i:nth-of-type(2) { left: 72%; top: 58px; }
|
||
.tree-mini em:nth-of-type(1) { left: 17%; bottom: 3px; }
|
||
.tree-mini em:nth-of-type(2) { left: 39%; bottom: 3px; }
|
||
.tree-mini em:nth-of-type(3) { left: 61%; bottom: 3px; }
|
||
.tree-mini em:nth-of-type(4) { left: 83%; bottom: 3px; }
|
||
|
||
.adaptive-mini { display: grid; gap: 9px; margin-top: 26px; }
|
||
.adaptive-mini b,
|
||
.adaptive-mini i { padding: 12px; background: var(--ink); color: var(--paper-raised); font: 0.64rem/1 var(--mono); }
|
||
.adaptive-mini i { width: 72%; background: var(--line-strong); color: var(--ink); }
|
||
.adaptive-mini i:last-child { width: 52%; }
|
||
|
||
.seq2seq-diagram {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1fr;
|
||
max-width: 1000px;
|
||
margin: 32px 0;
|
||
align-items: center;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.encoder-side,
|
||
.decoder-side { padding: 28px; }
|
||
.seq2seq-diagram > strong { display: grid; width: 64px; height: 64px; place-items: center; border: 2px solid var(--copper); color: var(--copper); font: 1.1rem/1 var(--mono); }
|
||
.encoder-side > div,
|
||
.decoder-side > div { display: flex; gap: 9px; align-items: center; margin: 24px 0 12px; }
|
||
.seq2seq-diagram b { padding: 8px; border: 1px solid var(--line); font-size: 0.72rem; }
|
||
.seq2seq-diagram i { color: var(--copper); }
|
||
.seq2seq-diagram small { color: var(--muted); font-size: 0.63rem; }
|
||
|
||
.plain-list {
|
||
max-width: 860px;
|
||
margin: 22px 0 30px;
|
||
padding: 0;
|
||
border-top: 1px solid var(--line);
|
||
list-style: none;
|
||
}
|
||
|
||
.plain-list li {
|
||
padding: 14px 0 14px 25px;
|
||
border-bottom: 1px solid var(--line);
|
||
color: var(--muted);
|
||
font-size: 0.75rem;
|
||
line-height: 1.6;
|
||
}
|
||
|
||
.plain-list li::before {
|
||
margin-left: -22px;
|
||
margin-right: 12px;
|
||
color: var(--copper);
|
||
content: "→";
|
||
}
|
||
|
||
.reverse-visual {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 920px;
|
||
margin: 26px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.reverse-visual > div { padding: 23px; }
|
||
.reverse-visual > div + div { border-left: 1px solid var(--line); }
|
||
.reverse-visual code { display: block; margin: 15px 0; font-size: 0.7rem; }
|
||
.reverse-visual i { display: block; height: 6px; background: var(--copper); }
|
||
.reverse-visual i.long { width: 90%; }
|
||
.reverse-visual i.short { width: 42%; }
|
||
.reverse-visual p { margin-top: 8px; color: var(--muted); font-size: 0.64rem; }
|
||
|
||
.attention-equations {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 980px;
|
||
margin: 32px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.attention-equations > div { padding: 24px; border-right: 1px solid var(--line); }
|
||
.attention-equations > div:last-child { border-right: 0; }
|
||
.attention-equations code { display: block; margin: 16px 0; color: var(--ink); font-size: 0.8rem; }
|
||
.attention-equations p { color: var(--muted); font-size: 0.68rem; }
|
||
|
||
.alignment-table {
|
||
display: grid;
|
||
grid-template-columns: 110px repeat(4, 1fr);
|
||
max-width: 760px;
|
||
margin: 30px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.alignment-table > div,
|
||
.alignment-table > i {
|
||
display: grid;
|
||
min-height: 62px;
|
||
place-items: center;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.alignment-table .corner,
|
||
.alignment-table .head,
|
||
.alignment-table .target {
|
||
color: var(--muted);
|
||
font: 0.61rem/1.3 var(--mono);
|
||
}
|
||
|
||
.alignment-table i {
|
||
background: color-mix(in srgb, var(--copper) calc(var(--a) * 100%), var(--paper));
|
||
}
|
||
|
||
.economics-grid {
|
||
grid-template-columns: repeat(4, 1fr);
|
||
}
|
||
|
||
.ppl-nonlinear {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto 1fr;
|
||
max-width: 940px;
|
||
margin: 30px 0;
|
||
align-items: center;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.ppl-nonlinear > div { display: grid; gap: 10px; padding: 24px; }
|
||
.ppl-nonlinear > i { color: var(--copper); font: 0.7rem/1 var(--mono); }
|
||
.ppl-nonlinear b { font: 1.2rem/1 var(--mono); }
|
||
.ppl-nonlinear em { color: var(--muted); font-size: 0.66rem; }
|
||
.ppl-nonlinear > p { grid-column: 1 / -1; padding: 12px 20px; border-top: 1px solid var(--line); color: var(--muted); font-size: 0.66rem; }
|
||
|
||
.parallel-compare {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 920px;
|
||
margin: 28px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.parallel-compare article { padding: 25px; }
|
||
.parallel-compare article + article { border-left: 1px solid var(--line); }
|
||
.parallel-compare article > div { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 20px 0; }
|
||
.parallel-compare b { padding: 8px; border: 1px solid var(--line); font: 0.66rem/1 var(--mono); }
|
||
.parallel-compare i { color: var(--copper); }
|
||
.parallel-compare strong { width: 100%; color: var(--copper); letter-spacing: 1.7em; text-align: center; }
|
||
.parallel-compare em { width: 100%; padding: 8px; background: var(--ink); color: var(--paper-raised); text-align: center; font: 0.66rem/1 var(--mono); }
|
||
.parallel-compare p { color: var(--muted); font-size: 0.68rem; }
|
||
|
||
.objective-stack {
|
||
display: grid;
|
||
grid-template-columns: 1.15fr auto 1fr auto 1fr auto 1fr;
|
||
max-width: 1080px;
|
||
margin: 34px 0;
|
||
align-items: stretch;
|
||
}
|
||
|
||
.objective-stack > div {
|
||
padding: 22px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.objective-stack > div.main {
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.objective-stack h3 { margin: 15px 0 10px; font-size: 1rem; }
|
||
.objective-stack code { color: var(--copper); font-size: 0.66rem; }
|
||
.objective-stack p { margin-top: 12px; color: var(--muted); font-size: 0.64rem; line-height: 1.55; }
|
||
.objective-stack > i { align-self: center; padding: 0 8px; color: var(--copper); }
|
||
|
||
.mtp-diagram {
|
||
display: grid;
|
||
grid-template-columns: 1.2fr 1fr;
|
||
max-width: 900px;
|
||
margin: 28px 0;
|
||
gap: 12px;
|
||
}
|
||
|
||
.mtp-diagram > div {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto auto;
|
||
gap: 12px;
|
||
align-items: center;
|
||
padding: 22px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.mtp-diagram span,
|
||
.mtp-diagram small { grid-column: 1 / -1; }
|
||
.mtp-diagram b { padding: 8px; background: var(--paper-deep); font: 0.7rem/1 var(--mono); }
|
||
.mtp-diagram i { color: var(--copper); }
|
||
.mtp-diagram strong { color: var(--copper); font: 0.8rem/1 var(--mono); }
|
||
.mtp-diagram small { color: var(--muted); font-size: 0.62rem; }
|
||
.mtp-diagram .mtp { border-color: var(--copper); }
|
||
|
||
.k3-token-stream {
|
||
display: grid;
|
||
grid-template-columns: auto auto auto auto 1fr;
|
||
max-width: 980px;
|
||
margin: 28px 0;
|
||
align-items: center;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.k3-token-stream > div { display: grid; gap: 8px; min-height: 94px; align-content: center; padding: 18px; border-right: 1px solid var(--line); }
|
||
.k3-token-stream .vision { background: var(--paper-deep); }
|
||
.k3-token-stream b { font-size: 0.75rem; }
|
||
.k3-token-stream > i { padding: 0 15px; color: var(--copper); }
|
||
.k3-token-stream > strong { padding: 0 15px; font-size: 0.82rem; }
|
||
.k3-token-stream > em { grid-column: 5; padding: 0 15px 16px; color: var(--copper); font: 0.62rem/1 var(--mono); }
|
||
|
||
.modern-contract {
|
||
max-width: 1040px;
|
||
margin: 30px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
|
||
.modern-contract > div {
|
||
display: grid;
|
||
grid-template-columns: 150px 1fr 1fr;
|
||
}
|
||
|
||
.modern-contract > div > * {
|
||
padding: 15px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.modern-contract .head { background: var(--ink); color: var(--paper-raised); }
|
||
.modern-contract span { color: var(--copper); font: 0.63rem/1.4 var(--mono); }
|
||
.modern-contract p { color: var(--muted); font-size: 0.68rem; }
|
||
|
||
.deepseek-highlight {
|
||
max-width: 980px;
|
||
margin: 38px 0;
|
||
padding: 30px;
|
||
border: 1px solid var(--copper);
|
||
background: color-mix(in srgb, var(--copper) 6%, var(--paper));
|
||
}
|
||
|
||
.deepseek-highlight h3 { max-width: 800px; margin: 14px 0; font-size: 1.35rem; }
|
||
.deepseek-highlight p { max-width: 840px; color: var(--muted); font-size: 0.73rem; line-height: 1.75; }
|
||
.deepseek-highlight a { display: inline-block; margin-top: 18px; color: var(--copper); font: 0.68rem/1 var(--mono); }
|
||
|
||
.causal-chain {
|
||
display: grid;
|
||
grid-template-columns: 1fr;
|
||
max-width: 920px;
|
||
margin: 42px 0;
|
||
}
|
||
|
||
.causal-chain article {
|
||
display: grid;
|
||
grid-template-columns: 45px 240px 1fr;
|
||
gap: 18px;
|
||
align-items: center;
|
||
padding: 18px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.causal-chain article.final { border-color: var(--copper); background: var(--ink); color: var(--paper-raised); }
|
||
.causal-chain h3 { font-size: 0.95rem; }
|
||
.causal-chain p { color: var(--muted); font-size: 0.68rem; }
|
||
.causal-chain > i { height: 28px; padding-left: 23px; color: var(--copper); font-style: normal; }
|
||
|
||
.checkpoint-list {
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
max-width: 1000px;
|
||
margin: 28px 0;
|
||
padding: 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
list-style: none;
|
||
}
|
||
|
||
.checkpoint-list li {
|
||
display: grid;
|
||
grid-template-columns: 38px 1fr;
|
||
gap: 12px;
|
||
min-height: 92px;
|
||
padding: 17px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.checkpoint-list span { color: var(--copper); font: 0.62rem/1.4 var(--mono); }
|
||
.checkpoint-list p { font-size: 0.72rem; line-height: 1.6; }
|
||
|
||
.next-chapter {
|
||
display: grid;
|
||
grid-template-columns: 1fr auto;
|
||
gap: 30px;
|
||
align-items: center;
|
||
max-width: 1000px;
|
||
margin: 42px 0;
|
||
padding: 30px;
|
||
background: var(--ink);
|
||
color: var(--paper-raised);
|
||
}
|
||
|
||
.next-chapter h3 { margin: 12px 0 8px; font-size: 1.3rem; }
|
||
.next-chapter p { color: color-mix(in srgb, var(--paper-raised) 65%, transparent); font-size: 0.7rem; }
|
||
.next-chapter a { padding: 13px 16px; border: 1px solid var(--copper); color: var(--copper); font: 0.66rem/1 var(--mono); white-space: nowrap; }
|
||
|
||
.paper-legend {
|
||
display: flex;
|
||
gap: 22px;
|
||
margin: 24px 0;
|
||
}
|
||
|
||
.paper-legend span {
|
||
display: flex;
|
||
gap: 8px;
|
||
align-items: center;
|
||
color: var(--muted);
|
||
font-size: 0.67rem;
|
||
}
|
||
|
||
.paper-chain {
|
||
max-width: 1100px;
|
||
border-top: 1px solid var(--line);
|
||
}
|
||
|
||
.paper-chain a {
|
||
display: grid;
|
||
grid-template-columns: 38px 58px minmax(280px, 1.7fr) minmax(180px, 1fr) 72px;
|
||
gap: 15px;
|
||
align-items: center;
|
||
min-height: 78px;
|
||
padding: 12px 10px;
|
||
border-bottom: 1px solid var(--line);
|
||
color: var(--ink);
|
||
}
|
||
|
||
.paper-chain a:hover { background: var(--paper-deep); }
|
||
.paper-chain a.core { border-left: 3px solid var(--copper); }
|
||
.paper-chain a.now { border-left: 3px solid var(--ink); background: color-mix(in srgb, var(--copper) 5%, var(--paper)); }
|
||
.paper-index,
|
||
.paper-chain time,
|
||
.paper-chain > a > b { font: 0.62rem/1 var(--mono); }
|
||
.paper-index { color: var(--muted-light); }
|
||
.paper-chain time { color: var(--copper); }
|
||
.paper-chain h3 { font-size: 0.82rem; line-height: 1.35; }
|
||
.paper-chain p,
|
||
.paper-chain em { margin-top: 4px; color: var(--muted); font-size: 0.65rem; }
|
||
.paper-chain > a > b { color: var(--copper); text-align: right; }
|
||
|
||
.research-ledger-link {
|
||
display: grid;
|
||
grid-template-columns: 150px 1fr auto;
|
||
gap: 24px;
|
||
align-items: center;
|
||
max-width: 1040px;
|
||
margin: 34px 0;
|
||
padding: 24px;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.research-ledger-link p { color: var(--muted); font-size: 0.68rem; line-height: 1.65; }
|
||
.research-ledger-link code { color: var(--ink); }
|
||
.research-ledger-link a { color: var(--copper); font: 0.66rem/1 var(--mono); white-space: nowrap; }
|
||
|
||
@media (max-width: 1000px) {
|
||
.ledger-grid,
|
||
.unit-comparison { grid-template-columns: repeat(2, 1fr); }
|
||
.unit-comparison article:nth-child(2) { border-right: 0; }
|
||
.unit-comparison article:nth-child(-n + 2) { border-bottom: 1px solid var(--line); }
|
||
.chain-visual { grid-template-columns: 1fr; gap: 7px; }
|
||
.chain-visual > i { justify-self: center; transform: rotate(90deg); }
|
||
.nplm-diagram { grid-template-columns: 1fr; gap: 8px; }
|
||
.nplm-diagram > i { justify-self: center; transform: rotate(90deg); }
|
||
.objective-stack { grid-template-columns: repeat(2, 1fr); }
|
||
.objective-stack > i { display: none; }
|
||
.paper-chain a { grid-template-columns: 35px 55px 1fr 80px; }
|
||
.paper-chain em { grid-column: 3; }
|
||
.paper-chain > a > b { grid-column: 4; grid-row: 1 / span 2; }
|
||
}
|
||
|
||
@media (max-width: 760px) {
|
||
.ledger-grid,
|
||
.reading-contract,
|
||
.prediction-demo,
|
||
.concept-split,
|
||
.three-choices,
|
||
.metric-ladder,
|
||
.unit-comparison,
|
||
.ngram-window,
|
||
.smoothing-pipeline,
|
||
.continuation-compare,
|
||
.evidence-row,
|
||
.representation-boundary,
|
||
.bottleneck-grid,
|
||
.word2vec-compare,
|
||
.negative-sampling,
|
||
.fixed-window-vs-state,
|
||
.gradient-regimes,
|
||
.gate-row,
|
||
.still-open,
|
||
.claim-contract,
|
||
.softmax-routes,
|
||
.boundary-pair,
|
||
.economics-grid,
|
||
.attention-equations,
|
||
.parallel-compare,
|
||
.mtp-diagram,
|
||
.checkpoint-list {
|
||
grid-template-columns: 1fr;
|
||
}
|
||
|
||
.ledger-grid article,
|
||
.unit-comparison article,
|
||
.reading-contract > div,
|
||
.ngram-window > div,
|
||
.concept-split article,
|
||
.three-choices > div,
|
||
.metric-ladder article,
|
||
.continuation-compare article,
|
||
.representation-boundary > div,
|
||
.word2vec-compare article,
|
||
.negative-sampling > div,
|
||
.fixed-window-vs-state article,
|
||
.softmax-routes article,
|
||
.boundary-pair article,
|
||
.economics-grid article,
|
||
.attention-equations > div,
|
||
.parallel-compare article {
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
|
||
.continuation-compare article + article,
|
||
.word2vec-compare article + article,
|
||
.fixed-window-vs-state article + article,
|
||
.reverse-visual > div + div,
|
||
.parallel-compare article + article {
|
||
border-left: 0;
|
||
border-top: 1px solid var(--line);
|
||
}
|
||
|
||
.source-boundary,
|
||
.history-correction,
|
||
.research-ledger-link {
|
||
grid-template-columns: 1fr;
|
||
}
|
||
|
||
.bpe-steps,
|
||
.sharing-staircase {
|
||
grid-template-columns: 1fr;
|
||
gap: 8px;
|
||
}
|
||
|
||
.bpe-steps > i,
|
||
.sharing-staircase > i {
|
||
justify-self: center;
|
||
transform: rotate(90deg);
|
||
}
|
||
|
||
.tokenizer-cost,
|
||
.tradeoff-axis,
|
||
.ppl-nonlinear,
|
||
.seq2seq-diagram {
|
||
grid-template-columns: 1fr;
|
||
}
|
||
|
||
.tokenizer-cost > i { justify-self: center; transform: rotate(90deg); }
|
||
.zero-chain { grid-template-columns: repeat(2, minmax(0, 1fr)); width: 100%; gap: 8px; }
|
||
.zero-chain > div { min-width: 0; }
|
||
.zero-chain > i { display: none; }
|
||
.zero-chain > strong { grid-column: 1 / -1; text-align: center; }
|
||
.smoothing-pipeline > i { justify-self: center; transform: rotate(90deg); }
|
||
.static-limit { grid-template-columns: 1fr; }
|
||
.static-limit > span,
|
||
.static-limit > div { border-right: 0; border-bottom: 1px solid var(--line); }
|
||
.static-limit > p { grid-column: 1; }
|
||
.rnn-unroll { grid-template-columns: 1fr; gap: 7px; }
|
||
.rnn-unroll > em { justify-self: center; padding: 0; transform: rotate(90deg); }
|
||
.cell-highway { grid-template-columns: auto 1fr auto 20px auto 20px auto; }
|
||
.reverse-visual { grid-template-columns: 1fr; }
|
||
.objective-stack { grid-template-columns: 1fr; }
|
||
.k3-token-stream { grid-template-columns: 1fr; }
|
||
.k3-token-stream > div { border-right: 0; border-bottom: 1px solid var(--line); }
|
||
.k3-token-stream > i { justify-self: center; transform: rotate(90deg); }
|
||
.k3-token-stream > strong,
|
||
.k3-token-stream > em { grid-column: 1; padding: 15px; text-align: center; }
|
||
.modern-contract > div { grid-template-columns: 95px 1fr 1fr; }
|
||
.causal-chain article { grid-template-columns: 34px 1fr; }
|
||
.causal-chain article p { grid-column: 2; }
|
||
.next-chapter { grid-template-columns: 1fr; }
|
||
.paper-chain a { grid-template-columns: 32px 52px 1fr; }
|
||
.paper-chain em { grid-column: 3; }
|
||
.paper-chain > a > b { grid-column: 3; grid-row: auto; text-align: left; }
|
||
}
|
||
|
||
@media (max-width: 520px) {
|
||
.ledger-grid { grid-template-columns: 1fr; }
|
||
.era-map > div { grid-template-columns: 82px 28px 1fr; }
|
||
.worked-example { grid-template-columns: 1fr; }
|
||
.worked-example > div + div { border-left: 0; border-top: 1px solid var(--line); }
|
||
.myth-grid { grid-template-columns: 1fr; }
|
||
.backoff-tree { grid-template-columns: 1fr; }
|
||
.backoff-tree > div:first-child { grid-column: 1; }
|
||
.backoff-tree .branch + .branch { border-left: 0; border-top: 1px solid var(--line); }
|
||
.alignment-table { grid-template-columns: 82px repeat(4, 1fr); }
|
||
.alignment-table > div,
|
||
.alignment-table > i { min-height: 50px; }
|
||
.modern-contract { width: 100%; max-width: 100%; overflow: hidden; }
|
||
.modern-contract > div { min-width: 0; grid-template-columns: 85px minmax(0, 1fr); }
|
||
.modern-contract > div > :nth-child(3) { grid-column: 2; }
|
||
.paper-legend { flex-direction: column; gap: 8px; }
|
||
.paper-chain a { grid-template-columns: 32px 1fr; gap: 8px; }
|
||
.paper-chain time { grid-column: 1; }
|
||
.paper-chain div,
|
||
.paper-chain em,
|
||
.paper-chain > a > b { grid-column: 2; }
|
||
.paper-chain > a > b { grid-row: auto; }
|
||
}
|
||
</style>
|
||
</BaseLayout>
|