feat: publish language model origins chapter

2026-07-29 04:47:36 +08:00
parent 2997434f7f
commit a54152dc24
22 changed files with 5521 additions and 35 deletions
@@ -35,9 +35,9 @@ export const chapters: Chapter[] = [
    kicker: "LANGUAGE MODELING",
    question: "预测下一个 Token，为什么能产生通用能力？",
    summary: "从 N-gram、神经概率语言模型、词向量一路走到 Seq2Seq，理解 Transformer 出现前的瓶颈。",
-    status: "researching",
-    progress: 24,
-    papers: 10,
+    status: "published",
+    progress: 78,
+    papers: 33,
    prerequisites: [],
    highlights: ["概率分解", "分布式表示", "序列瓶颈"],
  },
@@ -33,6 +33,102 @@ export const papers: Paper[] = [
    contribution: "信息熵与序列概率的理论起点。",
    verified: true,
  },
+  {
+    year: 1951,
+    title: "Prediction and Entropy of Printed English",
+    url: "https://doi.org/10.1002/j.1538-7305.1951.tb01366.x",
+    topics: ["基础"],
+    contribution: "用已知前文时的下一字母猜测实验估计英语熵与冗余，是 next-symbol 预测的概念先声。",
+    verified: true,
+  },
+  {
+    year: 1953,
+    title: "The Population Frequencies of Species and the Estimation of Population Parameters",
+    url: "https://doi.org/10.1093/biomet/40.3-4.237",
+    topics: ["基础"],
+    contribution: "以 frequency of frequencies 估计低频与未见事件的概率质量，奠定 Good–Turing 主线。",
+    verified: true,
+  },
+  {
+    year: 1977,
+    title: "Perplexity—a measure of the difficulty of speech recognition tasks",
+    url: "https://doi.org/10.1121/1.2016299",
+    topics: ["基础", "评测"],
+    contribution: "早期把 perplexity 用作语音识别语言约束难度的历史节点。",
+    verified: true,
+  },
+  {
+    year: 1987,
+    title: "Estimation of Probabilities from Sparse Data for the Language Model Component of a Speech Recognizer",
+    url: "https://ieeexplore.ieee.org/document/1165125",
+    topics: ["基础"],
+    contribution: "用折扣与回退把高阶 N-gram 的未见概率质量交给低阶模型。",
+    verified: true,
+  },
+  {
+    year: 1990,
+    title: "Finding Structure in Time",
+    url: "https://doi.org/10.1207/s15516709cog1402_1",
+    topics: ["基础"],
+    contribution: "Elman 网络用循环 context units 在预测任务中形成序列相关的隐藏结构。",
+    verified: true,
+  },
+  {
+    year: 1990,
+    title: "Indexing by Latent Semantic Analysis",
+    url: "https://doi.org/10.1002/(SICI)1097-4571(199009)41:6%3C391::AID-ASI1%3E3.0.CO;2-9",
+    topics: ["基础"],
+    contribution: "用词—文档矩阵的低秩分解构造全局潜在语义空间。",
+    verified: true,
+  },
+  {
+    year: 1992,
+    title: "Class-Based n-gram Models of Natural Language",
+    url: "https://aclanthology.org/J92-4003/",
+    topics: ["基础"],
+    contribution: "用离散词类在 N-gram 中共享统计，是连续词向量之前的重要参数共享路线。",
+    verified: true,
+  },
+  {
+    year: 1994,
+    title: "Learning long-term dependencies with gradient descent is difficult",
+    url: "https://ieeexplore.ieee.org/document/279181",
+    topics: ["基础"],
+    contribution: "系统阐明循环网络学习长期存储时的梯度与动力学困难。",
+    verified: true,
+  },
+  {
+    year: 1995,
+    title: "Improved Backing-off for M-gram Language Modeling",
+    url: "https://doi.org/10.1109/ICASSP.1995.479394",
+    topics: ["基础"],
+    contribution: "用不同续接上下文的数量构造低阶分布，形成 Kneser–Ney smoothing 的核心直觉。",
+    verified: true,
+  },
+  {
+    year: 1996,
+    title: "An Empirical Study of Smoothing Techniques for Language Modeling",
+    url: "https://aclanthology.org/P96-1041/",
+    topics: ["基础", "评测"],
+    contribution: "跨数据规模和模型阶数系统比较平滑方法，建立 modified Kneser–Ney 的实证坐标。",
+    verified: true,
+  },
+  {
+    year: 1997,
+    title: "Long Short-Term Memory",
+    url: "https://doi.org/10.1162/neco.1997.9.8.1735",
+    topics: ["基础"],
+    contribution: "用受门控的恒定误差通路缓解循环网络的长期依赖与衰减误差回流。",
+    verified: true,
+  },
+  {
+    year: 2001,
+    title: "A Bit of Progress in Language Modeling",
+    url: "https://arxiv.org/abs/cs/0108005",
+    topics: ["基础", "评测"],
+    contribution: "组合 cache、高阶 N-gram、skipping、Kneser–Ney 与 clustering，展现计数时代的强系统基线。",
+    verified: true,
+  },
  {
    year: 2003,
    title: "A Neural Probabilistic Language Model",
@@ -41,6 +137,22 @@ export const papers: Paper[] = [
    contribution: "用分布式词表示和神经网络突破 N-gram 稀疏泛化。",
    verified: true,
  },
+  {
+    year: 2005,
+    title: "Hierarchical Probabilistic Neural Network Language Model",
+    url: "https://proceedings.mlr.press/r5/morin05a.html",
+    topics: ["基础"],
+    contribution: "用二叉词树把全词表预测改成路径二分类，直面神经 LM 的 Softmax 词表税。",
+    verified: true,
+  },
+  {
+    year: 2010,
+    title: "Recurrent neural network based language model",
+    url: "https://www.isca-archive.org/interspeech_2010/mikolov10_interspeech.html",
+    topics: ["基础", "评测"],
+    contribution: "把变长循环状态带入真实语音识别 LM，并明确暴露高训练复杂度。",
+    verified: true,
+  },
  {
    year: 2013,
    title: "Efficient Estimation of Word Representations in Vector Space",
@@ -49,6 +161,38 @@ export const papers: Paper[] = [
    contribution: "Word2Vec 把大规模词向量训练变得简单高效。",
    verified: true,
  },
+  {
+    year: 2013,
+    title: "Distributed Representations of Words and Phrases and their Compositionality",
+    url: "https://proceedings.neurips.cc/paper/2013/hash/9aa42b31882ec039965f3c4923ce901b-Abstract.html",
+    topics: ["基础"],
+    contribution: "以 negative sampling、频繁词子采样和短语发现扩展高效静态词表示。",
+    verified: true,
+  },
+  {
+    year: 2013,
+    title: "One Billion Word Benchmark for Measuring Progress in Statistical Language Modeling",
+    url: "https://arxiv.org/abs/1312.3005",
+    topics: ["基础", "评测"],
+    contribution: "提供接近十亿词的可比 LM 基准，并并列计数、循环与组合系统。",
+    verified: true,
+  },
+  {
+    year: 2014,
+    title: "GloVe: Global Vectors for Word Representation",
+    url: "https://aclanthology.org/D14-1162/",
+    topics: ["基础"],
+    contribution: "在全局共现计数上拟合加权 log-bilinear 关系，连接矩阵分解与局部窗口表示。",
+    verified: true,
+  },
+  {
+    year: 2014,
+    title: "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation",
+    url: "https://aclanthology.org/D14-1179/",
+    topics: ["基础"],
+    contribution: "引入 RNN encoder–decoder 与 GRU 门控，把变长源序列映射到条件生成。",
+    verified: true,
+  },
  {
    year: 2014,
    title: "Sequence to Sequence Learning with Neural Networks",
@@ -65,6 +209,14 @@ export const papers: Paper[] = [
    contribution: "用可微软对齐让 decoder 直接检索全部 encoder state。",
    verified: true,
  },
+  {
+    year: 2015,
+    title: "Effective Approaches to Attention-based Neural Machine Translation",
+    url: "https://aclanthology.org/D15-1166/",
+    topics: ["基础", "Transformer"],
+    contribution: "系统化 global/local attention 与多种打分形式，澄清 Attention 前夜的设计空间。",
+    verified: true,
+  },
  {
    year: 2015,
    title: "Neural Machine Translation of Rare Words with Subword Units",
@@ -73,6 +225,46 @@ export const papers: Paper[] = [
    contribution: "把 BPE 引入神经机器翻译，形成现代子词分词主线。",
    verified: true,
  },
+  {
+    year: 2016,
+    title: "Character-Aware Neural Language Models",
+    url: "https://arxiv.org/abs/1508.06615",
+    topics: ["基础"],
+    contribution: "用字符 CNN 与 highway network 构造词表示，再交给 LSTM LM 建模。",
+    verified: true,
+  },
+  {
+    year: 2016,
+    title: "Exploring the Limits of Language Modeling",
+    url: "https://arxiv.org/abs/1602.02410",
+    topics: ["基础", "训练系统", "评测"],
+    contribution: "在 Transformer 前夜把 RNN/CNN 词级语言模型推向大数据、大词表与大计算。",
+    verified: true,
+  },
+  {
+    year: 2016,
+    title: "Efficient softmax approximation for GPUs",
+    url: "https://arxiv.org/abs/1609.04309",
+    topics: ["基础", "训练系统"],
+    contribution: "Adaptive Softmax 利用词频长尾分配不同计算，降低全词表输出的期望成本。",
+    verified: true,
+  },
+  {
+    year: 2016,
+    title: "Language Modeling with Gated Convolutional Networks",
+    url: "https://arxiv.org/abs/1612.08083",
+    topics: ["基础", "训练系统"],
+    contribution: "以门控卷积建立可并行的强语言模型路线，说明历史并非 RNN 单线演进。",
+    verified: true,
+  },
+  {
+    year: 2017,
+    title: "Regularizing and Optimizing LSTM Language Models",
+    url: "https://arxiv.org/abs/1708.02182",
+    topics: ["基础", "训练系统"],
+    contribution: "以 DropConnect 与 NT-ASGD 等配方建立 AWD-LSTM 强基线。",
+    verified: true,
+  },
  {
    year: 1991,
    title: "Adaptive Mixtures of Local Experts",
@@ -474,7 +666,7 @@ export const papers: Paper[] = [
    year: 2024,
    title: "DeepSeek-V3 Technical Report",
    url: "https://arxiv.org/abs/2412.19437",
-    topics: ["MoE", "Scaling", "数据", "训练系统", "低精度"],
+    topics: ["基础", "MoE", "Scaling", "数据", "训练系统", "低精度"],
    contribution: "671B-A37B；FP8、MTP、无辅助损失平衡与 DualPipe。",
    spotlight: "DeepSeek",
    verified: true,
@@ -1814,7 +2006,7 @@ export const papers: Paper[] = [
    year: 2026,
    title: "DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence",
    url: "https://arxiv.org/abs/2606.19348",
-    topics: ["长上下文", "MoE", "数据", "训练系统", "推理"],
+    topics: ["基础", "长上下文", "MoE", "数据", "训练系统", "推理"],
    contribution: "32/33T 长文与 agentic 数据、CSA/HCA、mHC、Muon 和 1M 上下文双模型。",
    spotlight: "DeepSeek",
    verified: true,
@@ -1823,7 +2015,7 @@ export const papers: Paper[] = [
    year: 2026,
    title: "Kimi K3: Open Frontier Intelligence",
    url: "https://arxiv.org/abs/2607.24653",
-    topics: ["长上下文", "MoE", "数据", "训练系统", "低精度", "推理", "Agent", "多模态", "评测"],
+    topics: ["基础", "长上下文", "MoE", "数据", "训练系统", "低精度", "推理", "Agent", "多模态", "评测"],
    contribution: "四文本域与视觉语料、2.8T-A104B、KDA/MLA、Stable LatentMoE 与 1M Agentic RL。",
    spotlight: "Kimi",
    verified: true,