feat: deepen attention and transformer chapter
This commit is contained in:
wuyang
+1619
-477
File diff suppressed because it is too large
Load Diff
+28
-1
@@ -81,6 +81,7 @@ const paths = [
|
||||
<div class="hero-actions">
|
||||
<a class="button primary" href="/roadmap/">选择学习路径 <span aria-hidden="true">↓</span></a>
|
||||
<a class="button" href="/foundations/language-models/">语言模型从哪里来</a>
|
||||
<a class="button" href="/foundations/">注意力与 Transformer</a>
|
||||
<a class="button" href="/k3/">直接解剖 K3</a>
|
||||
<a class="button" href="/deepseek/">DeepSeek 专题</a>
|
||||
<a class="button" href="/scaling/">Scaling Laws 专题</a>
|
||||
@@ -99,7 +100,7 @@ const paths = [
|
||||
<div class="hero-stats">
|
||||
<div><b>16</b><span>核心专题</span></div>
|
||||
<div><b>151</b><span>K3 报告来源</span></div>
|
||||
<div><b>247</b><span>关键论文索引</span></div>
|
||||
<div><b>258</b><span>关键论文索引</span></div>
|
||||
<div><b>47p</b><span>K3 技术报告</span></div>
|
||||
</div>
|
||||
</aside>
|
||||
@@ -113,6 +114,22 @@ const paths = [
|
||||
|
||||
<section class="section compact release-section" id="new-chapters">
|
||||
<div class="release-grid">
|
||||
<a class="release-card transformer-release" href="/foundations/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / CHAPTER 02</span> ATTENTION · TRANSFORMER</p>
|
||||
<h2>Attention 不只是“看哪里”，Transformer 也不只有一种 Block</h2>
|
||||
<p>
|
||||
用信息路径、匹配几何、可见性、多头、位置、局部计算、深度、架构目标、系统成本与当代映射十张账，
|
||||
从 Bahdanau 软对齐和 2017 原始 encoder–decoder，一路走到 FlashAttention、DeepSeek MLA 与 Kimi K3。
|
||||
</p>
|
||||
</div>
|
||||
<dl>
|
||||
<div><dt>LINEAGE</dt><dd>2014 → 2026</dd></div>
|
||||
<div><dt>PAPERS</dt><dd>40 个一手节点</dd></div>
|
||||
<div><dt>LAB</dt><dd>QKV · Mask · 多头位置 · Block 成本</dd></div>
|
||||
</dl>
|
||||
<span class="release-arrow" aria-hidden="true">从软对齐进入现代 LLM 骨架 →</span>
|
||||
</a>
|
||||
<a class="release-card foundation-release" href="/foundations/language-models/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / CHAPTER 01</span> LANGUAGE MODELING ORIGINS</p>
|
||||
@@ -474,6 +491,7 @@ const paths = [
|
||||
transition: transform 180ms ease, border-color 180ms ease;
|
||||
}
|
||||
|
||||
.transformer-release,
|
||||
.foundation-release,
|
||||
.numerics-release,
|
||||
.data-release,
|
||||
@@ -500,6 +518,14 @@ const paths = [
|
||||
var(--paper-raised);
|
||||
}
|
||||
|
||||
.transformer-release {
|
||||
background:
|
||||
radial-gradient(circle at 82% 18%, rgba(56, 91, 128, 0.21), transparent 30%),
|
||||
radial-gradient(circle at 60% 74%, rgba(159, 91, 52, 0.12), transparent 26%),
|
||||
repeating-linear-gradient(90deg, transparent 0 64px, rgba(56, 91, 128, 0.04) 64px 65px),
|
||||
var(--paper-raised);
|
||||
}
|
||||
|
||||
.data-release {
|
||||
background:
|
||||
radial-gradient(circle at 82% 18%, rgba(76, 118, 112, 0.2), transparent 30%),
|
||||
@@ -590,6 +616,7 @@ const paths = [
|
||||
padding-bottom: 76px;
|
||||
}
|
||||
|
||||
.transformer-release,
|
||||
.foundation-release,
|
||||
.numerics-release,
|
||||
.data-release,
|
||||
|
||||
@@ -11,7 +11,7 @@ const workstreams = [
|
||||
{ label: "网站设计系统", value: 89, next: "打印样式与更多通用可视化组件" },
|
||||
{ label: "Kimi K3 深读", value: 66, next: "扩写 pre-training / infra 逐图笔记" },
|
||||
{ label: "语言模型前史", value: 78, next: "逐图精读 Kneser–Ney、LSTM 与 Bahdanau，并加入真实小语料复现" },
|
||||
{ label: "Transformer 基础", value: 52, next: "加入矩阵形状动画与手算练习" },
|
||||
{ label: "Transformer 基础", value: 79, next: "逐图精读多头电路、Pre/Post-LN 与真实 kernel / KV 配置" },
|
||||
{ label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" },
|
||||
{ label: "数据工程与预训练配方", value: 73, next: "逐图精读 FineWeb / DCLM，加入真实去重与 mixture traces" },
|
||||
{ label: "DeepSeek 专题", value: 71, next: "补 R1 / DAPO 的逐图训练轨迹与复现对照" },
|
||||
@@ -44,7 +44,7 @@ const workstreams = [
|
||||
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
|
||||
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
|
||||
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 04:48 CST</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 05:20 CST</dd></div>
|
||||
<div><dt>MODE</dt><dd>持续迭代，不锁死版本</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
@@ -91,9 +91,10 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
|
||||
<article><span>✓</span><h3>16 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
|
||||
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
|
||||
<article><span>✓</span><h3>二十八个原创交互视图</h3><p>K3、语言模型前史、注意力、DeepSeek、长上下文、MoE、推理，以及训练系统、Scaling、数据工程和数值专题的多页签实验。</p></article>
|
||||
<article><span>✓</span><h3>三十一个原创交互视图</h3><p>K3、语言模型前史、Transformer 四联实验、DeepSeek、长上下文、MoE、推理，以及训练系统、Scaling、数据工程和数值专题。</p></article>
|
||||
<article><span>✓</span><h3>十一篇首版长文</h3><p>K3、语言模型前史、Transformer、DeepSeek、Scaling、数据工程、长上下文、MoE、推理、训练系统与数值优化专题。</p></article>
|
||||
<article><span>✓</span><h3>语言模型前史深度专题</h3><p>八张独立问题账、33 个正式节点、20 段长文与概率—向量—记忆—对齐四联实验。</p></article>
|
||||
<article><span>✓</span><h3>Transformer 深度专题</h3><p>十张独立问题账、40 个正式节点、21 段正文与 QKV—Mask—多头位置—Block 成本四联实验。</p></article>
|
||||
<article><span>✓</span><h3>Scaling Laws 深度专题</h3><p>九张账、29 个一手节点、DeepSeek/Kimi 双谱系与曲面—部署—复用—涌现四联实验。</p></article>
|
||||
<article><span>✓</span><h3>数据工程深度专题</h3><p>十二张账、31 个一手节点、DeepSeek/Kimi 双谱系与流水线—去重—混合—改写四联实验。</p></article>
|
||||
<article><span>✓</span><h3>长上下文深度专题</h3><p>五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。</p></article>
|
||||
@@ -101,7 +102,7 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>推理深度专题</h3><p>八张账、30 篇一手论文链、DeepSeek/Kimi 双主线与三页签互动实验室。</p></article>
|
||||
<article><span>✓</span><h3>训练系统深度专题</h3><p>九张账、37 个一手节点、DeepSeek/Kimi 双谱系与显存—网格—气泡—通信实验室。</p></article>
|
||||
<article><span>✓</span><h3>数值、优化器与稳定性深度专题</h3><p>十张账、36 个一手节点、K2/K3 与 DeepSeek-V3/V4 双谱系，以及格式—状态—更新—失稳四联实验。</p></article>
|
||||
<article><span>✓</span><h3>247 篇关键论文索引</h3><p>新增 24 个语言模型前史节点，并把 DeepSeek-V3/V4 与 K3 映射回 next-token 主线。</p></article>
|
||||
<article><span>✓</span><h3>258 篇关键论文索引</h3><p>补齐 Residual/Norm、相对位置、UniLM、多头冗余、Attention 解释争论与 NormFormer 等 11 个节点。</p></article>
|
||||
<article><span>✓</span><h3>公开仓库与自托管发布</h3><p>源码公开到 git.k1412.top，网站由不可变镜像、Compose Manager 与 HTTPS 交付。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
@@ -116,6 +117,7 @@ const workstreams = [
|
||||
</div>
|
||||
<div class="queue-table">
|
||||
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
|
||||
<div><span>P0</span><strong>Transformer 二轮</strong><p>多头电路逐图 → Pre/Post-LN 真实 traces → Flash/KV 配置与 kernel 对照</p><em>逐图笔记 + 实测边界</em></div>
|
||||
<div><span>P0</span><strong>语言模型前史二轮</strong><p>Kneser–Ney / LSTM / Bahdanau 逐图 → 真实小语料复现 → tokenizer 公平性</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P0</span><strong>Scaling Laws 二轮</strong><p>真实拟合复现 → 置信区间 → 更多模型族与下游任务外推</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P1</span><strong>数据工程二轮</strong><p>FineWeb / DCLM 逐图 → 真实去重误伤 → mixture traces 与污染案例</p><em>逐图笔记 + 案例库</em></div>
|
||||
@@ -172,6 +174,8 @@ const workstreams = [
|
||||
<div><time>2026-07-29</time><b>低精度结论必须写完整角色合同</b><p>对象、格式、scale 粒度、accumulator、输出与硬件不再被压缩成一个 dtype 标签。</p></div>
|
||||
<div><time>2026-07-29</time><b>语言模型前史按八张账组织</b><p>预测单位、概率信息、上下文、稀疏性、分布式表示、循环记忆、序列转导与系统成本不再混成单一架构年表。</p></div>
|
||||
<div><time>2026-07-29</time><b>NTP、MTP 与多模态永久分角色</b><p>K3 的统一视觉/文本 next-token objective、one MTP layer 与 EAGLE-3 draft bridge 分开记账；DeepSeek MTP 也不写成取代自回归。</p></div>
|
||||
<div><time>2026-07-29</time><b>Transformer 按十张账组织</b><p>信息路径、几何、可见性、多头、位置、局部计算、深度、目标、系统成本与当代映射不再混成一个 Block。</p></div>
|
||||
<div><time>2026-07-29</time><b>Attention 权重与因果解释永久分离</b><p>热力图可描述中间权重和提出假设；因果结论必须补消融、patching 或反事实干预。</p></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user