feat: publish representation and depth chapter
This commit is contained in:
wuyang
@@ -0,0 +1,893 @@
|
||||
---
|
||||
const tokenPresets = {
|
||||
word: ["我", "在", "河岸", "看见", "一座", "银行"],
|
||||
subword: ["我", "在", "河", "岸", "看见", "一", "座", "银", "行"],
|
||||
byte: ["E6", "88", "91", "E5", "9C", "A8", "E6", "B2", "B3", "…", "E8", "A1", "8C"],
|
||||
patch: ["[我在]", "[河岸]", "[看见]", "[一座银行]"],
|
||||
};
|
||||
---
|
||||
|
||||
<figure class="rep-lab" data-representation-lab>
|
||||
<div class="rep-lab-head">
|
||||
<div>
|
||||
<p>INTERACTIVE / REPRESENTATION WORKBENCH</p>
|
||||
<h3>沿四个坐标轴拆开一条 hidden state</h3>
|
||||
</div>
|
||||
<p>
|
||||
四台仪器只计算公开公式与明确标注的 toy model。它们帮助建立方向直觉，不是任何真实 tokenizer、
|
||||
checkpoint、训练 loss 或梯度的复跑。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="rep-tabs" role="tablist" aria-label="选择表示与深度实验">
|
||||
<button type="button" role="tab" data-rep-tab="token" aria-selected="true">
|
||||
<span>01</span><b>Token 与表示</b><small>unit · tying · context</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-rep-tab="position" aria-selected="false" tabindex="-1">
|
||||
<span>02</span><b>位置几何</b><small>RoPE · ALiBi · NoPE</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-rep-tab="norm" aria-selected="false" tabindex="-1">
|
||||
<span>03</span><b>Norm 与深度</b><small>topology · scale · logits</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-rep-tab="residual" aria-selected="false" tabindex="-1">
|
||||
<span>04</span><b>Residual 与 FFN</b><small>mHC · AttnRes · SiTU</small>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<section class="rep-view" data-rep-panel="token">
|
||||
<div class="rep-panel-intro">
|
||||
<div><span>WORKBENCH 01 / REPRESENTATION</span><h4>一个“词”进模型后，至少会经过五种身份</h4></div>
|
||||
<p>切换计算单位、词表和 weight tying，观察参数账；再让同一 token 进入两个上下文，看初始向量如何被改写。</p>
|
||||
</div>
|
||||
|
||||
<div class="rep-controls">
|
||||
<label>
|
||||
<span>计算单位</span>
|
||||
<select data-token-unit>
|
||||
<option value="word">教学词级</option>
|
||||
<option value="subword" selected>教学子词</option>
|
||||
<option value="byte">UTF-8 byte</option>
|
||||
<option value="patch">动态 byte patch</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
<span>词表 V</span>
|
||||
<select data-vocab>
|
||||
<option value="32000">32K</option>
|
||||
<option value="128000">128K</option>
|
||||
<option value="160000" selected>160K / K3</option>
|
||||
<option value="256000">256K</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
<span>hidden width d <output data-width-label>7,168</output></span>
|
||||
<input data-width type="range" min="512" max="8192" step="256" value="7168" />
|
||||
</label>
|
||||
<label class="check-control">
|
||||
<input data-weight-tying type="checkbox" checked />
|
||||
<span><b>共享输入 / 输出矩阵</b><small>weight tying</small></span>
|
||||
</label>
|
||||
<label>
|
||||
<span>上下文</span>
|
||||
<select data-context>
|
||||
<option value="river" selected>河岸边的银行</option>
|
||||
<option value="finance">办理金融业务</option>
|
||||
<option value="none">只看初始 embedding</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="token-pipeline" aria-label="token 表示流水线">
|
||||
<div><span>RAW</span><b>“我在河岸看见一座银行”</b><small>字符串不是模型一步</small></div>
|
||||
<i aria-hidden="true">→</i>
|
||||
<div><span>SEGMENT</span><b data-token-count>9 tokens</b><small data-token-ratio>0.64 token / 字</small></div>
|
||||
<i aria-hidden="true">→</i>
|
||||
<div><span>LOOKUP</span><b>ID → E[id]</b><small>固定的第 0 层向量</small></div>
|
||||
<i aria-hidden="true">→</i>
|
||||
<div class="active"><span>CONTEXT</span><b>h<sup>l</sup><sub>token</sub></b><small>随句子、位置、层而变</small></div>
|
||||
<i aria-hidden="true">→</i>
|
||||
<div><span>LOGITS</span><b>W<sub>out</sub>h</b><small>再成为词表分类分数</small></div>
|
||||
</div>
|
||||
|
||||
<div class="token-chips" data-token-chips aria-label="教学分词结果">
|
||||
{tokenPresets.subword.map((token, index) => <span><small>{index}</small>{token}</span>)}
|
||||
</div>
|
||||
|
||||
<div class="rep-metrics four">
|
||||
<article><span>SEQUENCE STEPS</span><b data-token-steps>9</b><p>同一句话的教学计算步数</p></article>
|
||||
<article><span>EMBEDDING PARAMS</span><b data-embedding-params>1.15B</b><p>`V × d`，不含位置和 block</p></article>
|
||||
<article><span>INPUT + OUTPUT</span><b data-io-params>1.15B</b><p>共享时一张表；否则两张</p></article>
|
||||
<article class="dark"><span>BF16 FOOTPRINT</span><b data-embedding-memory>2.14 GiB</b><p>只算 input / output matrix</p></article>
|
||||
</div>
|
||||
|
||||
<div class="vector-stage">
|
||||
<div class="vector-copy">
|
||||
<span>SAME TOKEN / DIFFERENT STATE</span>
|
||||
<h5>初始 embedding 不读上下文，hidden state 会</h5>
|
||||
<p data-vector-explain>“行”在河岸语境中被教学变换到“行走/景物”方向；这不是任何真实模型的向量，也不宣称二维投影能容纳全部语义。</p>
|
||||
<dl>
|
||||
<div><dt>BASE E[id]</dt><dd data-base-vector>(0.24, 0.61)</dd></div>
|
||||
<div><dt>CONTEXT hˡ</dt><dd data-context-vector>(−0.43, 0.78)</dd></div>
|
||||
<div><dt>SHIFT</dt><dd data-vector-shift>0.71</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
<svg viewBox="0 0 440 280" role="img" aria-label="初始向量和上下文化向量的二维教学投影">
|
||||
<line x1="40" y1="140" x2="410" y2="140" />
|
||||
<line x1="220" y1="22" x2="220" y2="258" />
|
||||
<circle cx="263" cy="68" r="8" class="base-point" />
|
||||
<line x1="220" y1="140" x2="263" y2="68" class="base-line" />
|
||||
<circle data-context-point cx="143" cy="48" r="9" class="context-point" />
|
||||
<line data-context-line x1="220" y1="140" x2="143" y2="48" class="context-line" />
|
||||
<text x="274" y="66">E[id]</text>
|
||||
<text data-context-name x="94" y="39">river hˡ</text>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="boundary-note">
|
||||
<b>先记住</b>
|
||||
<p data-token-boundary>Tokenizer 决定计算单位，embedding 只负责入口查表；真正的上下文化表示来自后续层的序列混合与 FFN。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="rep-view" data-rep-panel="position" hidden>
|
||||
<div class="rep-panel-intro">
|
||||
<div><span>WORKBENCH 02 / POSITION</span><h4>位置不是一个数字，而是它进入匹配公式的方式</h4></div>
|
||||
<p>移动 Q / K 的位置，比较绝对相加、RoPE 旋转、ALiBi 距离偏置与没有显式项的 NoPE。</p>
|
||||
</div>
|
||||
|
||||
<div class="position-schemes" role="group" aria-label="选择位置方案">
|
||||
<button type="button" data-position-scheme="absolute"><b>ABSOLUTE</b><small>内容 + 位置向量</small></button>
|
||||
<button type="button" data-position-scheme="rope" class="active"><b>RoPE</b><small>旋转 Q / K</small></button>
|
||||
<button type="button" data-position-scheme="alibi"><b>ALiBi</b><small>logit 距离惩罚</small></button>
|
||||
<button type="button" data-position-scheme="nope"><b>NoPE</b><small>无显式位置项</small></button>
|
||||
</div>
|
||||
|
||||
<div class="rep-controls">
|
||||
<label>
|
||||
<span>Query 位置 i <output data-query-position-label>24</output></span>
|
||||
<input data-query-position type="range" min="0" max="256" value="24" />
|
||||
</label>
|
||||
<label>
|
||||
<span>Key 位置 j <output data-key-position-label>8</output></span>
|
||||
<input data-key-position type="range" min="0" max="256" value="8" />
|
||||
</label>
|
||||
<label>
|
||||
<span>训练窗口</span>
|
||||
<select data-train-window>
|
||||
<option value="32">32</option>
|
||||
<option value="64" selected>64</option>
|
||||
<option value="128">128</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
<span>RoPE 频率 θ <output data-frequency-label>0.040</output></span>
|
||||
<input data-frequency type="range" min="5" max="100" value="40" />
|
||||
</label>
|
||||
<label>
|
||||
<span>ALiBi slope <output data-slope-label>0.060</output></span>
|
||||
<input data-slope type="range" min="0" max="200" value="60" />
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="position-stage">
|
||||
<svg viewBox="0 0 470 330" role="img" aria-label="位置方案对 Q K 几何的教学影响">
|
||||
<circle cx="235" cy="165" r="112" class="orbit" />
|
||||
<line x1="48" y1="165" x2="422" y2="165" />
|
||||
<line x1="235" y1="24" x2="235" y2="306" />
|
||||
<line data-q-arrow x1="235" y1="165" x2="336" y2="123" class="q-arrow" />
|
||||
<circle data-q-point cx="336" cy="123" r="9" class="q-point" />
|
||||
<line data-k-arrow x1="235" y1="165" x2="314" y2="86" class="k-arrow" />
|
||||
<circle data-k-point cx="314" cy="86" r="9" class="k-point" />
|
||||
<path data-angle-arc d="M 280 145 A 48 48 0 0 0 270 130" class="angle-arc" />
|
||||
<text data-q-caption x="347" y="122">Q@24</text>
|
||||
<text data-k-caption x="325" y="82">K@8</text>
|
||||
<text x="20" y="25">2D TEACHING FREQUENCY</text>
|
||||
</svg>
|
||||
<div class="position-readout">
|
||||
<span data-position-mode>ROTATE CONTENT VECTORS</span>
|
||||
<b data-position-equation>qᵀ R(j−i) k</b>
|
||||
<p data-position-explain>RoPE 让两次绝对旋转在点积中相消，只留下相对位移；这里仅画一个二维频率对。</p>
|
||||
<div class="window-line">
|
||||
<i data-window-fill style="width:37.5%"></i>
|
||||
<b data-window-marker style="left:37.5%"></b>
|
||||
</div>
|
||||
<small data-window-status>两位置都在训练窗口 0–64 内</small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="rep-metrics four">
|
||||
<article><span>RELATIVE Δ</span><b data-relative-distance>−16</b><p>`j − i`，方向也属于信息</p></article>
|
||||
<article><span>EXPLICIT TERM</span><b data-explicit-term>−0.640 rad</b><p>当前方案写入匹配的量</p></article>
|
||||
<article><span>TOY SCORE</span><b data-position-score>0.552</b><p>单个二维内容向量的示意</p></article>
|
||||
<article class="dark"><span>REGIME</span><b data-position-regime>INTERPOLATION</b><p>可计算不等于训练分布熟悉</p></article>
|
||||
</div>
|
||||
|
||||
<div class="position-compare">
|
||||
<article><span>ABSOLUTE</span><b>先混进 state</b><p>每个位置有自己的向量；相对规律需由网络再组合。</p></article>
|
||||
<article><span>RELATIVE / ALiBi</span><b>直接改 pair score</b><p>距离在 Q–K 匹配时才出现，不必污染 Value。</p></article>
|
||||
<article><span>RoPE</span><b>旋转后做点积</b><p>绝对角度进入 Q/K，相对角度留在内积。</p></article>
|
||||
<article><span>NoPE</span><b>显式项为 0</b><p>不能据此断言 causal / recurrent model 没学顺序。</p></article>
|
||||
</div>
|
||||
|
||||
<div class="boundary-note">
|
||||
<b>外推边界</b>
|
||||
<p data-position-boundary>当前位置仍在训练窗口内；把任一位置移到 64 之外，就进入 toy extrapolation 区。公式仍能算，但相位组合未必训练过。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="rep-view" data-rep-panel="norm" hidden>
|
||||
<div class="rep-panel-intro">
|
||||
<div><span>WORKBENCH 03 / NORMALIZATION</span><h4>先问 Norm 控制谁，再问它放在哪里</h4></div>
|
||||
<p>同一条 residual branch，Norm 的对象和位置会改变 identity path、累计幅值和 attention logit 饱和。</p>
|
||||
</div>
|
||||
|
||||
<div class="norm-topologies" role="group" aria-label="选择归一化拓扑">
|
||||
<button type="button" data-norm-topology="post"><b>POST-LN</b><small>Add → Norm</small></button>
|
||||
<button type="button" data-norm-topology="pre" class="active"><b>PRE-LN</b><small>Norm → Branch → Add</small></button>
|
||||
<button type="button" data-norm-topology="rms"><b>RMS-PRE</b><small>只控制 RMS</small></button>
|
||||
<button type="button" data-norm-topology="qk"><b>QK + RMS</b><small>state 与 logits 分治</small></button>
|
||||
</div>
|
||||
|
||||
<div class="rep-controls">
|
||||
<label>
|
||||
<span>网络深度 L <output data-depth-label>96</output></span>
|
||||
<input data-depth type="range" min="12" max="192" step="12" value="96" />
|
||||
</label>
|
||||
<label>
|
||||
<span>分支 RMS s <output data-branch-label>0.18</output></span>
|
||||
<input data-branch-scale type="range" min="2" max="60" value="18" />
|
||||
</label>
|
||||
<label>
|
||||
<span>原始 QK logit <output data-logit-label>8.0</output></span>
|
||||
<input data-logit-scale type="range" min="5" max="160" value="80" />
|
||||
</label>
|
||||
<label>
|
||||
<span>可竞争 keys</span>
|
||||
<select data-key-count>
|
||||
<option value="4">4</option>
|
||||
<option value="8" selected>8</option>
|
||||
<option value="32">32</option>
|
||||
<option value="128">128</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="norm-diagram">
|
||||
<div class="identity-rail">
|
||||
<span>IDENTITY PATH</span>
|
||||
<i></i><b data-identity-label>不经过 Norm，直接跨层</b><i></i>
|
||||
</div>
|
||||
<div class="norm-blocks">
|
||||
<div data-norm-node>RMS / LN</div><i>→</i><div>ATTN / FFN</div><i>→</i><div class="add-node">＋</div>
|
||||
</div>
|
||||
<p data-norm-equation>xₗ₊₁ = xₗ + F(Norm(xₗ))</p>
|
||||
</div>
|
||||
|
||||
<div class="depth-chart">
|
||||
<div class="depth-chart-head"><span>TOY HIDDEN-STATE RMS OVER DEPTH</span><b data-final-rms>2.03×</b></div>
|
||||
<div class="depth-bars" data-depth-bars aria-label="不同深度的 toy hidden-state RMS"></div>
|
||||
<div class="depth-axis"><span>0</span><span>L / 4</span><span>L / 2</span><span>3L / 4</span><span>L</span></div>
|
||||
</div>
|
||||
|
||||
<div class="rep-metrics four">
|
||||
<article><span>FINAL TOY RMS</span><b data-norm-rms>2.03×</b><p>假设分支近似正交的教学估计</p></article>
|
||||
<article><span>IDENTITY GAIN</span><b data-identity-gain>1.00</b><p>捷径上的局部教学系数</p></article>
|
||||
<article><span>SOFTMAX PEAK</span><b data-softmax-peak>99.8%</b><p>一个 logit 高于其余 0 的 toy 分布</p></article>
|
||||
<article class="dark"><span>NORM TARGET</span><b data-norm-target>hidden state</b><p>位置与对象不能只看名字</p></article>
|
||||
</div>
|
||||
|
||||
<div class="norm-objects">
|
||||
<article><span>LAYERNORM</span><b>中心 + 尺度</b><code>(x−μ) / √(σ²+ε)</code></article>
|
||||
<article><span>RMSNORM</span><b>只管尺度</b><code>x / √mean(x²)</code></article>
|
||||
<article><span>QK-NORM</span><b>只管匹配向量</b><code>q̂ · k̂ × γ</code></article>
|
||||
<article><span>LATENT NORM</span><b>只管压缩状态</b><code>Norm(c<sub>KV</sub>)</code></article>
|
||||
</div>
|
||||
|
||||
<div class="boundary-note">
|
||||
<b>这不是训练曲线</b>
|
||||
<p data-norm-boundary>Pre-LN 保留裸 identity path；toy RMS 用独立分支近似展示固定相加为何可能随深度增长，不能预测真实 checkpoint。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="rep-view" data-rep-panel="residual" hidden>
|
||||
<div class="rep-panel-intro">
|
||||
<div><span>WORKBENCH 04 / DEPTH & FFN</span><h4>一条状态既要穿过深度，也要穿过乘法门控</h4></div>
|
||||
<p>左侧选择深度路径，右侧选择激活；把 DeepSeek-V4 的 mHC / clamp 与 K3 的 AttnRes / SiTU 放在同一张机制图中。</p>
|
||||
</div>
|
||||
|
||||
<div class="residual-grid">
|
||||
<div class="residual-workbench">
|
||||
<div class="subhead"><span>DEPTH ROUTING</span><h5>谁能进入当前层？</h5></div>
|
||||
<div class="route-buttons" role="group" aria-label="选择残差路径">
|
||||
<button type="button" data-route="plain"><b>PLAIN</b><small>固定单位相加</small></button>
|
||||
<button type="button" data-route="hc"><b>HC</b><small>4 条 residual streams</small></button>
|
||||
<button type="button" data-route="mhc"><b>mHC</b><small>受约束的流间映射</small></button>
|
||||
<button type="button" data-route="attnres" class="active"><b>ATTNRES</b><small>沿深度 softmax</small></button>
|
||||
</div>
|
||||
<div class="mini-controls">
|
||||
<label><span>depth L <output data-route-depth-label>93</output></span><input data-route-depth type="range" min="24" max="120" value="93" /></label>
|
||||
<label><span>当前内容偏好 <output data-route-focus-label>6</output></span><input data-route-focus type="range" min="0" max="8" value="6" /></label>
|
||||
</div>
|
||||
<div class="depth-sources" data-depth-sources aria-label="深度来源权重"></div>
|
||||
<div class="route-equation">
|
||||
<span data-route-name>BLOCK ATTENTION RESIDUALS</span>
|
||||
<code data-route-equation>hₗ = Σ softmax(qₗᵀ RMSNorm(kᵢ)) · vᵢ</code>
|
||||
<p data-route-explain>K3 把 93 层按 12 层组织为 8 个 layer blocks；加上 embedding，共有 9 个 block-level 来源。</p>
|
||||
</div>
|
||||
<div class="rep-metrics two">
|
||||
<article><span>LIVE DEPTH STATES</span><b data-live-states>9</b><p>教学状态数；不是显存字节</p></article>
|
||||
<article class="dark"><span>DEPTH CHOICE</span><b data-depth-choice>CONTENT-DEPENDENT</b><p>固定相加还是按内容选择</p></article>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="activation-workbench">
|
||||
<div class="subhead"><span>FFN ACTIVATION</span><h5>乘法门控怎样处理极值？</h5></div>
|
||||
<div class="activation-buttons" role="group" aria-label="选择激活函数">
|
||||
<button type="button" data-activation="gelu"><b>GELU</b><small>平滑单支</small></button>
|
||||
<button type="button" data-activation="swiglu"><b>SwiGLU</b><small>两支无界相乘</small></button>
|
||||
<button type="button" data-activation="clamp"><b>V4 CLAMP</b><small>硬限制两支</small></button>
|
||||
<button type="button" data-activation="situ" class="active"><b>K3 SiTU</b><small>平滑有界</small></button>
|
||||
</div>
|
||||
<div class="mini-controls">
|
||||
<label><span>输入范围 ±x <output data-input-range-label>40</output></span><input data-input-range type="range" min="2" max="80" value="40" /></label>
|
||||
</div>
|
||||
<svg class="activation-chart" viewBox="0 0 520 280" role="img" aria-label="激活函数极值响应的教学曲线">
|
||||
<line x1="34" y1="140" x2="504" y2="140" />
|
||||
<line x1="260" y1="18" x2="260" y2="262" />
|
||||
<path data-activation-path d="" />
|
||||
<text x="36" y="25">OUTPUT / NORMALIZED VIEW</text>
|
||||
<text x="472" y="160">+x</text>
|
||||
<text x="238" y="276">0</text>
|
||||
</svg>
|
||||
<div class="activation-formula">
|
||||
<span data-activation-name>KIMI K3 / SiTU-GLU</span>
|
||||
<code data-activation-equation>4 tanh(g/4) · σ(g) · 25 tanh(u/25)</code>
|
||||
<p data-activation-explain>两条线性支路都用 tanh 平滑压缩；β₁=4、β₂=25 时，逐点绝对值严格小于 100。</p>
|
||||
</div>
|
||||
<div class="rep-metrics two">
|
||||
<article><span>MAX |Y| / RANGE</span><b data-activation-max>85.7</b><p>在当前绘图区间采样</p></article>
|
||||
<article class="dark"><span>GLOBAL BOUND</span><b data-activation-bound>< 100</b><p>函数性质或“无有限上界”</p></article>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="model-contrast">
|
||||
<article class="deepseek-card">
|
||||
<span>DEEPSEEK-V4 / 2026</span>
|
||||
<h5>扩宽深度路，再把它投回稳定流形</h5>
|
||||
<ul>
|
||||
<li>mHC residual expansion = 4</li>
|
||||
<li>Birkhoff polytope / doubly stochastic B</li>
|
||||
<li>Q / compressed KV 各做 head-wise RMSNorm</li>
|
||||
<li>最后 64 维 partial RoPE</li>
|
||||
<li>SwiGLU linear `[-10,10]`、gate upper cap `10`</li>
|
||||
</ul>
|
||||
</article>
|
||||
<div aria-hidden="true">≠</div>
|
||||
<article class="kimi-card">
|
||||
<span>KIMI K3 / 2026</span>
|
||||
<h5>沿深度做注意力，再给乘法输出平滑上界</h5>
|
||||
<ul>
|
||||
<li>Block AttnRes over 9 depth sources</li>
|
||||
<li>RMSNorm(keys) 抵消幅值支配</li>
|
||||
<li>KDA 隐式位置 + MLA NoPE</li>
|
||||
<li>routed latent 后额外 RMSNorm</li>
|
||||
<li>SiTU-GLU：β₁β₂ = 100 bound</li>
|
||||
</ul>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="boundary-note">
|
||||
<b>不要排成一个排行榜</b>
|
||||
<p data-residual-boundary>mHC 学习受约束的多流映射；AttnRes 对先前层或 block 表示做内容相关选择。二者改变的是不同拓扑，不能只用“谁更先进”概括。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<figcaption>
|
||||
<span>教学模拟</span>
|
||||
公式和机制来自已核验论文；二维向量、RMS 曲线、source scores 与采样范围由本站构造，目的是让变量关系可操作。
|
||||
</figcaption>
|
||||
</figure>
|
||||
|
||||
<script>
|
||||
const roots = document.querySelectorAll<HTMLElement>("[data-representation-lab]");
|
||||
roots.forEach((root) => {
|
||||
const one = <T extends Element>(selector: string) => root.querySelector<T>(selector);
|
||||
const all = <T extends Element>(selector: string) => [...root.querySelectorAll<T>(selector)];
|
||||
const number = (selector: string) => Number((one<HTMLInputElement | HTMLSelectElement>(selector))?.value ?? 0);
|
||||
const set = (selector: string, value: string) => {
|
||||
const node = one<HTMLElement>(selector);
|
||||
if (node) node.textContent = value;
|
||||
};
|
||||
const compact = (value: number) => value >= 1e9 ? `${(value / 1e9).toFixed(2)}B` : value >= 1e6 ? `${(value / 1e6).toFixed(1)}M` : value.toLocaleString();
|
||||
const sigmoid = (x: number) => 1 / (1 + Math.exp(-x));
|
||||
|
||||
const tabs = all<HTMLButtonElement>("[data-rep-tab]");
|
||||
const panels = all<HTMLElement>("[data-rep-panel]");
|
||||
const selectTab = (tab: HTMLButtonElement) => {
|
||||
tabs.forEach((candidate) => {
|
||||
const selected = candidate === tab;
|
||||
candidate.setAttribute("aria-selected", String(selected));
|
||||
candidate.tabIndex = selected ? 0 : -1;
|
||||
});
|
||||
panels.forEach((panel) => panel.hidden = panel.dataset.repPanel !== tab.dataset.repTab);
|
||||
};
|
||||
tabs.forEach((tab, index) => {
|
||||
tab.addEventListener("click", () => selectTab(tab));
|
||||
tab.addEventListener("keydown", (event) => {
|
||||
if (!["ArrowLeft", "ArrowRight", "Home", "End"].includes(event.key)) return;
|
||||
event.preventDefault();
|
||||
let target = index;
|
||||
if (event.key === "ArrowRight") target = (index + 1) % tabs.length;
|
||||
if (event.key === "ArrowLeft") target = (index - 1 + tabs.length) % tabs.length;
|
||||
if (event.key === "Home") target = 0;
|
||||
if (event.key === "End") target = tabs.length - 1;
|
||||
tabs[target].focus();
|
||||
selectTab(tabs[target]);
|
||||
});
|
||||
});
|
||||
|
||||
const tokenSets: Record<string, string[]> = {
|
||||
word: ["我", "在", "河岸", "看见", "一座", "银行"],
|
||||
subword: ["我", "在", "河", "岸", "看见", "一", "座", "银", "行"],
|
||||
byte: ["E6", "88", "91", "E5", "9C", "A8", "E6", "B2", "B3", "…", "E8", "A1", "8C"],
|
||||
patch: ["[我在]", "[河岸]", "[看见]", "[一座银行]"],
|
||||
};
|
||||
const contextVectors: Record<string, [number, number, string, string]> = {
|
||||
river: [-0.43, 0.78, "river hˡ", "“行”在河岸语境中被教学变换到“行走/景物”方向；这不是任何真实模型的向量，也不宣称二维投影能容纳全部语义。"],
|
||||
finance: [0.82, -0.16, "finance hˡ", "“行”在金融语境中被教学变换到“机构/交易”方向；初始 ID 相同，差异来自上下文计算。"],
|
||||
none: [0.24, 0.61, "E[id]", "关闭上下文化后，两点重合：初始 embedding 是 token ID 的固定查表结果，尚未读取句内其他 token。"],
|
||||
};
|
||||
const renderToken = () => {
|
||||
const unit = one<HTMLSelectElement>("[data-token-unit]")?.value ?? "subword";
|
||||
const vocab = number("[data-vocab]");
|
||||
const width = number("[data-width]");
|
||||
const tied = Boolean(one<HTMLInputElement>("[data-weight-tying]")?.checked);
|
||||
const context = one<HTMLSelectElement>("[data-context]")?.value ?? "river";
|
||||
const tokens = tokenSets[unit];
|
||||
const chars = 14;
|
||||
const embedding = vocab * width;
|
||||
const io = embedding * (tied ? 1 : 2);
|
||||
const vector = contextVectors[context];
|
||||
const base: [number, number] = [0.24, 0.61];
|
||||
const shift = Math.hypot(vector[0] - base[0], vector[1] - base[1]);
|
||||
const pointX = 220 + vector[0] * 180;
|
||||
const pointY = 140 - vector[1] * 118;
|
||||
|
||||
set("[data-width-label]", width.toLocaleString());
|
||||
set("[data-token-count]", `${tokens.length} tokens`);
|
||||
set("[data-token-ratio]", `${(tokens.length / chars).toFixed(2)} token / 字`);
|
||||
set("[data-token-steps]", String(tokens.length));
|
||||
set("[data-embedding-params]", compact(embedding));
|
||||
set("[data-io-params]", compact(io));
|
||||
set("[data-embedding-memory]", `${(io * 2 / 1024 ** 3).toFixed(2)} GiB`);
|
||||
set("[data-context-vector]", `(${vector[0].toFixed(2)}, ${vector[1].toFixed(2)})`);
|
||||
set("[data-vector-shift]", shift.toFixed(2));
|
||||
set("[data-vector-explain]", vector[3]);
|
||||
set("[data-context-name]", vector[2]);
|
||||
set("[data-token-boundary]", `${unit === "byte" ? "Byte 去掉固定子词边界，却增加序列步数；" : unit === "patch" ? "动态 patch 把可预测 byte 合并，计算单位不再等长；" : "Tokenizer 先决定模型的一步；"}当前 ${tied ? "共享" : "不共享"}输入 / 输出权重，因此这两张接口表占 ${compact(io)} 参数。`);
|
||||
const chips = one<HTMLElement>("[data-token-chips]");
|
||||
if (chips) chips.innerHTML = tokens.map((token, index) => `<span><small>${index}</small>${token}</span>`).join("");
|
||||
const point = one<SVGCircleElement>("[data-context-point]");
|
||||
const line = one<SVGLineElement>("[data-context-line]");
|
||||
const label = one<SVGTextElement>("[data-context-name]");
|
||||
point?.setAttribute("cx", String(pointX));
|
||||
point?.setAttribute("cy", String(pointY));
|
||||
line?.setAttribute("x2", String(pointX));
|
||||
line?.setAttribute("y2", String(pointY));
|
||||
label?.setAttribute("x", String(Math.max(8, pointX - 48)));
|
||||
label?.setAttribute("y", String(Math.max(16, pointY - 12)));
|
||||
};
|
||||
all<HTMLInputElement | HTMLSelectElement>("[data-token-unit],[data-vocab],[data-width],[data-weight-tying],[data-context]").forEach((control) => control.addEventListener("input", renderToken));
|
||||
|
||||
let positionScheme = "rope";
|
||||
const renderPosition = () => {
|
||||
const i = number("[data-query-position]");
|
||||
const j = number("[data-key-position]");
|
||||
const train = number("[data-train-window]");
|
||||
const frequency = number("[data-frequency]") / 1000;
|
||||
const slope = number("[data-slope]") / 1000;
|
||||
const delta = j - i;
|
||||
const baseQ = 0.35;
|
||||
const baseK = 0.72;
|
||||
let qAngle = baseQ;
|
||||
let kAngle = baseK;
|
||||
let term = "0";
|
||||
let equation = "qᵀk";
|
||||
let mode = "NO EXPLICIT POSITION TERM";
|
||||
let explanation = "NoPE 不给 Q/K 添加显式位置项；因果 mask、递归状态或训练数据仍可能提供隐式顺序。";
|
||||
let score = Math.cos(baseK - baseQ);
|
||||
if (positionScheme === "rope") {
|
||||
qAngle += i * frequency;
|
||||
kAngle += j * frequency;
|
||||
term = `${(delta * frequency).toFixed(3)} rad`;
|
||||
equation = "qᵀ R(j−i) k";
|
||||
mode = "ROTATE CONTENT VECTORS";
|
||||
explanation = "RoPE 让两次绝对旋转在点积中相消，只留下相对位移；这里仅画一个二维频率对。";
|
||||
score = Math.cos(kAngle - qAngle);
|
||||
} else if (positionScheme === "alibi") {
|
||||
term = `${(-slope * Math.abs(delta)).toFixed(3)} bias`;
|
||||
equation = "qᵀk − m·|j−i|";
|
||||
mode = "BIAS THE ATTENTION LOGIT";
|
||||
explanation = "ALiBi 不旋转向量，而是按距离给 attention logit 加线性惩罚；不同 heads 可用不同 slope。";
|
||||
score -= slope * Math.abs(delta);
|
||||
} else if (positionScheme === "absolute") {
|
||||
qAngle += i * frequency;
|
||||
kAngle += j * frequency * 1.17;
|
||||
term = `p${i} + p${j}`;
|
||||
equation = "(q+pᵢ)ᵀ(k+pⱼ)";
|
||||
mode = "ADD POSITION TO CONTENT";
|
||||
explanation = "绝对位置先与内容相加；网络可以再组合出相对规律，但内容与位置从入口开始共同改变 state。";
|
||||
score = Math.cos(kAngle - qAngle);
|
||||
}
|
||||
const point = (angle: number) => [235 + Math.cos(angle) * 112, 165 - Math.sin(angle) * 112];
|
||||
const q = point(qAngle);
|
||||
const k = point(kAngle);
|
||||
const outside = Math.max(i, j) > train;
|
||||
const marker = Math.min(100, Math.max(i, j) / 256 * 100);
|
||||
const fill = train / 256 * 100;
|
||||
|
||||
set("[data-query-position-label]", String(i));
|
||||
set("[data-key-position-label]", String(j));
|
||||
set("[data-frequency-label]", frequency.toFixed(3));
|
||||
set("[data-slope-label]", slope.toFixed(3));
|
||||
set("[data-relative-distance]", delta > 0 ? `+${delta}` : String(delta));
|
||||
set("[data-explicit-term]", term);
|
||||
set("[data-position-score]", score.toFixed(3));
|
||||
set("[data-position-regime]", outside ? "EXTRAPOLATION" : "INTERPOLATION");
|
||||
set("[data-position-mode]", mode);
|
||||
set("[data-position-equation]", equation);
|
||||
set("[data-position-explain]", explanation);
|
||||
set("[data-window-status]", outside ? `至少一个位置越过训练窗口 0–${train}` : `两位置都在训练窗口 0–${train} 内`);
|
||||
set("[data-position-boundary]", outside
|
||||
? `当前位置进入 toy extrapolation 区：公式仍能计算到 ${Math.max(i, j)}，但相位、bias 或距离组合可能从未在 0–${train} 训练分布出现。`
|
||||
: `当前位置仍在训练窗口内；把任一位置移到 ${train} 之外，就进入 toy extrapolation 区。公式仍能算，但分布未必熟悉。`);
|
||||
const fillNode = one<HTMLElement>("[data-window-fill]");
|
||||
const markerNode = one<HTMLElement>("[data-window-marker]");
|
||||
if (fillNode) fillNode.style.width = `${fill}%`;
|
||||
if (markerNode) markerNode.style.left = `${marker}%`;
|
||||
const assignPoint = (prefix: string, p: number[], caption: string) => {
|
||||
const arrow = one<SVGLineElement>(`[data-${prefix}-arrow]`);
|
||||
const dot = one<SVGCircleElement>(`[data-${prefix}-point]`);
|
||||
const text = one<SVGTextElement>(`[data-${prefix}-caption]`);
|
||||
arrow?.setAttribute("x2", String(p[0]));
|
||||
arrow?.setAttribute("y2", String(p[1]));
|
||||
dot?.setAttribute("cx", String(p[0]));
|
||||
dot?.setAttribute("cy", String(p[1]));
|
||||
text?.setAttribute("x", String(p[0] + 12));
|
||||
text?.setAttribute("y", String(p[1] - 6));
|
||||
if (text) text.textContent = caption;
|
||||
};
|
||||
assignPoint("q", q, `Q@${i}`);
|
||||
assignPoint("k", k, `K@${j}`);
|
||||
};
|
||||
all<HTMLButtonElement>("[data-position-scheme]").forEach((button) => button.addEventListener("click", () => {
|
||||
positionScheme = button.dataset.positionScheme ?? "rope";
|
||||
all("[data-position-scheme]").forEach((candidate) => candidate.classList.toggle("active", candidate === button));
|
||||
renderPosition();
|
||||
}));
|
||||
all<HTMLInputElement | HTMLSelectElement>("[data-query-position],[data-key-position],[data-train-window],[data-frequency],[data-slope]").forEach((control) => control.addEventListener("input", renderPosition));
|
||||
|
||||
let normTopology = "pre";
|
||||
const renderNorm = () => {
|
||||
const depth = number("[data-depth]");
|
||||
const branch = number("[data-branch-scale]") / 100;
|
||||
const logit = number("[data-logit-scale]") / 10;
|
||||
const keys = number("[data-key-count]");
|
||||
let finalRms = Math.sqrt(1 + depth * branch ** 2);
|
||||
let identity = 1;
|
||||
let target = "hidden state";
|
||||
let identityLabel = "不经过 Norm，直接跨层";
|
||||
let equation = "xₗ₊₁ = xₗ + F(Norm(xₗ))";
|
||||
let normNode = "LayerNorm";
|
||||
if (normTopology === "post") {
|
||||
finalRms = 1;
|
||||
identity = Math.exp(-depth / 180);
|
||||
identityLabel = "每次相加后都穿过 Norm";
|
||||
equation = "xₗ₊₁ = Norm(xₗ + F(xₗ))";
|
||||
target = "post-add state";
|
||||
} else if (normTopology === "rms") {
|
||||
finalRms = Math.sqrt(1 + depth * branch ** 2);
|
||||
normNode = "RMSNorm";
|
||||
target = "state RMS";
|
||||
} else if (normTopology === "qk") {
|
||||
finalRms = Math.sqrt(1 + depth * branch ** 2);
|
||||
normNode = "RMSNorm + QK";
|
||||
target = "state + Q/K";
|
||||
}
|
||||
const effectiveLogit = normTopology === "qk" ? Math.min(3.2, logit * 0.4) : logit;
|
||||
const peak = Math.exp(effectiveLogit) / (Math.exp(effectiveLogit) + keys - 1);
|
||||
set("[data-depth-label]", String(depth));
|
||||
set("[data-branch-label]", branch.toFixed(2));
|
||||
set("[data-logit-label]", logit.toFixed(1));
|
||||
set("[data-final-rms]", `${finalRms.toFixed(2)}×`);
|
||||
set("[data-norm-rms]", `${finalRms.toFixed(2)}×`);
|
||||
set("[data-identity-gain]", identity.toFixed(2));
|
||||
set("[data-softmax-peak]", `${(peak * 100).toFixed(1)}%`);
|
||||
set("[data-norm-target]", target);
|
||||
set("[data-identity-label]", identityLabel);
|
||||
set("[data-norm-equation]", equation);
|
||||
set("[data-norm-node]", normNode);
|
||||
set("[data-norm-boundary]", normTopology === "post"
|
||||
? "Post-LN 在每次相加后重新归一化；toy RMS 固定为 1，但 identity path 不再是裸恒等映射。"
|
||||
: normTopology === "qk"
|
||||
? "QK-Norm 直接压住匹配向量尺度，toy softmax 不再被大 logit 推到近 one-hot；它不取代 residual stream 的 RMSNorm。"
|
||||
: `${normTopology === "rms" ? "RMSNorm 不做去均值；" : "Pre-LN 保留裸 identity path；"}toy RMS 用独立分支近似展示固定相加为何可能随深度增长，不能预测真实 checkpoint。`);
|
||||
const bars = one<HTMLElement>("[data-depth-bars]");
|
||||
if (bars) {
|
||||
const values = Array.from({ length: 20 }, (_, index) => {
|
||||
const layer = depth * (index + 1) / 20;
|
||||
return normTopology === "post" ? 1 : Math.sqrt(1 + layer * branch ** 2);
|
||||
});
|
||||
const max = Math.max(...values, 1.05);
|
||||
bars.innerHTML = values.map((value, index) => `<i style="height:${Math.max(8, value / max * 100)}%" title="depth ${Math.round(depth * (index + 1) / 20)}: ${value.toFixed(2)}×"><span>${index === 19 ? value.toFixed(2) : ""}</span></i>`).join("");
|
||||
}
|
||||
};
|
||||
all<HTMLButtonElement>("[data-norm-topology]").forEach((button) => button.addEventListener("click", () => {
|
||||
normTopology = button.dataset.normTopology ?? "pre";
|
||||
all("[data-norm-topology]").forEach((candidate) => candidate.classList.toggle("active", candidate === button));
|
||||
renderNorm();
|
||||
}));
|
||||
all<HTMLInputElement | HTMLSelectElement>("[data-depth],[data-branch-scale],[data-logit-scale],[data-key-count]").forEach((control) => control.addEventListener("input", renderNorm));
|
||||
|
||||
let route = "attnres";
|
||||
let activation = "situ";
|
||||
const renderRoute = () => {
|
||||
const depth = number("[data-route-depth]");
|
||||
const focus = number("[data-route-focus]");
|
||||
let states = 1;
|
||||
let name = "STANDARD PRENORM RESIDUAL";
|
||||
let equation = "hₗ = h₀ + Σᵢ fᵢ(hᵢ)";
|
||||
let explain = "所有层输出用固定单位权重累加到同一条 residual stream；捷径直接，但选择性为零。";
|
||||
let choice = "FIXED UNIT SUM";
|
||||
let weights: number[] = Array.from({ length: 9 }, () => 1);
|
||||
if (route === "hc") {
|
||||
states = 4;
|
||||
name = "HYPER-CONNECTIONS";
|
||||
equation = "Xₗ₊₁ = BₗXₗ + CₗFₗ(AₗXₗ)";
|
||||
explain = "HC 把 residual width 展开为 4 条流，并学习输入、流间和输出映射；自由度增加，也可能破坏 identity property。";
|
||||
choice = "LEARNED MULTI-STREAM";
|
||||
weights = [0.46, 0.71, 0.34, 0.82, 0.58, 0.65, 0.39, 0.74, 0.52];
|
||||
} else if (route === "mhc") {
|
||||
states = 4;
|
||||
name = "MANIFOLD-CONSTRAINED HC";
|
||||
equation = "Bₗ ∈ Birkhoff polytope · ‖Bₗ‖₂ ≤ 1";
|
||||
explain = "mHC 保留 4 条 residual streams，但把流间 B 映射投影到 Birkhoff polytope，使其 doubly stochastic，限制深层连续映射的扩张。";
|
||||
choice = "CONSTRAINED MIXING";
|
||||
weights = [0.52, 0.61, 0.47, 0.66, 0.55, 0.58, 0.49, 0.63, 0.54];
|
||||
} else if (route === "attnres") {
|
||||
states = Math.ceil(depth / 12) + 1;
|
||||
name = "BLOCK ATTENTION RESIDUALS";
|
||||
equation = "hₗ = Σ softmax(qₗᵀ RMSNorm(kᵢ)) · vᵢ";
|
||||
explain = depth === 93
|
||||
? "K3 把 93 层按 12 层组织为 8 个 layer blocks；加上 embedding，共有 9 个 block-level 来源。"
|
||||
: `按 12 层教学分块后有 ${Math.ceil(depth / 12)} 个 layer blocks；加 embedding，共 ${states} 个 depth sources。`;
|
||||
choice = "CONTENT-DEPENDENT";
|
||||
weights = Array.from({ length: Math.min(11, states) }, (_, index) => Math.exp(-Math.abs(index - focus) * 0.62) * (1 + ((index * 7) % 4) * 0.08));
|
||||
}
|
||||
const total = weights.reduce((sum, value) => sum + value, 0);
|
||||
const normalized = weights.map((value) => value / total);
|
||||
const sources = one<HTMLElement>("[data-depth-sources]");
|
||||
if (sources) sources.innerHTML = normalized.map((value, index) => `<div class="${index === normalized.indexOf(Math.max(...normalized)) ? "selected" : ""}"><i style="height:${Math.max(7, value / Math.max(...normalized) * 100)}%"></i><b>${(value * 100).toFixed(0)}%</b><span>${index === 0 ? "EMB" : `B${index}`}</span></div>`).join("");
|
||||
set("[data-route-depth-label]", String(depth));
|
||||
set("[data-route-focus-label]", String(focus));
|
||||
set("[data-route-name]", name);
|
||||
set("[data-route-equation]", equation);
|
||||
set("[data-route-explain]", explain);
|
||||
set("[data-live-states]", String(states));
|
||||
set("[data-depth-choice]", choice);
|
||||
set("[data-residual-boundary]", route === "mhc"
|
||||
? "mHC 学习受约束的多流映射；它的 doubly stochastic 约束针对连续深度混合的稳定性，不等于沿历史层检索。"
|
||||
: route === "attnres"
|
||||
? "AttnRes 对先前层或 block 表示做内容相关选择；Full 与 Block 的状态成本不同，K3 使用 Block 版本。"
|
||||
: "标准 residual、HC、mHC 与 AttnRes 改变不同拓扑；本实验展示信息路径，不给它们排统一优劣。");
|
||||
};
|
||||
const gelu = (x: number) => 0.5 * x * (1 + Math.tanh(Math.sqrt(2 / Math.PI) * (x + 0.044715 * x ** 3)));
|
||||
const activationFn = (kind: string, x: number) => {
|
||||
if (kind === "gelu") return gelu(x);
|
||||
if (kind === "swiglu") return x * sigmoid(x) * x;
|
||||
if (kind === "clamp") return Math.min(x * sigmoid(x), 10) * Math.max(-10, Math.min(10, x));
|
||||
return 4 * Math.tanh(x / 4) * sigmoid(x) * 25 * Math.tanh(x / 25);
|
||||
};
|
||||
const renderActivation = () => {
|
||||
const range = number("[data-input-range]");
|
||||
const samples = Array.from({ length: 161 }, (_, index) => -range + 2 * range * index / 160);
|
||||
const values = samples.map((x) => activationFn(activation, x));
|
||||
const maxAbs = Math.max(...values.map(Math.abs), 1);
|
||||
const points = values.map((value, index) => {
|
||||
const x = 34 + index / 160 * 470;
|
||||
const y = 140 - value / maxAbs * 112;
|
||||
return `${index === 0 ? "M" : "L"}${x.toFixed(1)},${y.toFixed(1)}`;
|
||||
}).join(" ");
|
||||
one<SVGPathElement>("[data-activation-path]")?.setAttribute("d", points);
|
||||
const details: Record<string, [string, string, string, string]> = {
|
||||
gelu: ["GELU", "x · Φ(x)", "GELU 是平滑单分支激活；正方向随 x 无界，负方向趋近 0。", "UNBOUNDED"],
|
||||
swiglu: ["SwiGLU / TOY SAME INPUT", "Swish(g) · u", "SwiGLU 的 gate 与 linear 分支都可随输入增大；图中为 g=u=x 的教学切片。", "UNBOUNDED"],
|
||||
clamp: ["DEEPSEEK-V4 / CLAMP", "min(Swish(g),10) · clamp(u,−10,10)", "V4 报告对 linear 分支做 [−10,10] clamp，并把 gate 上界 cap 为 10；硬边界来自作者配方。", "≤ 100 / TOY"],
|
||||
situ: ["KIMI K3 / SiTU-GLU", "4 tanh(g/4) · σ(g) · 25 tanh(u/25)", "两条线性支路都用 tanh 平滑压缩；β₁=4、β₂=25 时，逐点绝对值严格小于 100。", "< 100"],
|
||||
};
|
||||
const detail = details[activation];
|
||||
set("[data-input-range-label]", String(range));
|
||||
set("[data-activation-name]", detail[0]);
|
||||
set("[data-activation-equation]", detail[1]);
|
||||
set("[data-activation-explain]", detail[2]);
|
||||
set("[data-activation-max]", maxAbs >= 1000 ? maxAbs.toExponential(2) : maxAbs.toFixed(1));
|
||||
set("[data-activation-bound]", detail[3]);
|
||||
};
|
||||
all<HTMLButtonElement>("[data-route]").forEach((button) => button.addEventListener("click", () => {
|
||||
route = button.dataset.route ?? "plain";
|
||||
all("[data-route]").forEach((candidate) => candidate.classList.toggle("active", candidate === button));
|
||||
renderRoute();
|
||||
}));
|
||||
all<HTMLButtonElement>("[data-activation]").forEach((button) => button.addEventListener("click", () => {
|
||||
activation = button.dataset.activation ?? "gelu";
|
||||
all("[data-activation]").forEach((candidate) => candidate.classList.toggle("active", candidate === button));
|
||||
renderActivation();
|
||||
}));
|
||||
all<HTMLInputElement>("[data-route-depth],[data-route-focus]").forEach((control) => control.addEventListener("input", renderRoute));
|
||||
one<HTMLInputElement>("[data-input-range]")?.addEventListener("input", renderActivation);
|
||||
|
||||
renderToken();
|
||||
renderPosition();
|
||||
renderNorm();
|
||||
renderRoute();
|
||||
renderActivation();
|
||||
});
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.rep-lab { margin: 36px 0; border: 1px solid var(--line-strong); background: var(--paper); box-shadow: 0 22px 70px rgba(8,18,30,.16); }
|
||||
.rep-lab-head { display:grid; grid-template-columns:1fr 1fr; gap:36px; align-items:end; padding:28px 30px; color:#f3eee5; background:linear-gradient(135deg,#172437,#24384e); }
|
||||
.rep-lab-head p:first-child,.rep-panel-intro span,.subhead span { margin:0 0 9px; color:#d49a68; font:600 .55rem var(--mono); letter-spacing:.13em; }
|
||||
.rep-lab-head h3 { margin:0; color:#fff; font:650 clamp(1.25rem,2.5vw,2rem) var(--serif); }
|
||||
.rep-lab-head > p { margin:0; color:#b7c1cc; font-size:.65rem; line-height:1.8; }
|
||||
.rep-tabs { display:grid; grid-template-columns:repeat(4,1fr); border-bottom:1px solid var(--line-strong); }
|
||||
.rep-tabs button { min-height:92px; display:grid; grid-template-columns:32px 1fr; grid-template-rows:auto auto; gap:4px 8px; padding:18px; border:0; border-right:1px solid var(--line); color:var(--ink); background:var(--paper-raised); text-align:left; cursor:pointer; }
|
||||
.rep-tabs button:last-child { border-right:0; }
|
||||
.rep-tabs button[aria-selected="true"] { color:#fff; background:var(--navy); }
|
||||
.rep-tabs span { grid-row:1/3; color:var(--copper); font:600 .6rem var(--mono); }
|
||||
.rep-tabs b { font:650 .84rem var(--serif); }
|
||||
.rep-tabs small { color:var(--muted); font:.49rem var(--mono); }
|
||||
.rep-tabs button[aria-selected="true"] small { color:#aab7c5; }
|
||||
.rep-view { padding:30px; }
|
||||
.rep-panel-intro { display:grid; grid-template-columns:1.1fr .9fr; gap:34px; align-items:end; margin-bottom:24px; }
|
||||
.rep-panel-intro h4 { margin:0; font:650 clamp(1.15rem,2.4vw,1.8rem) var(--serif); }
|
||||
.rep-panel-intro > p { margin:0; color:var(--muted); font-size:.65rem; line-height:1.75; }
|
||||
.rep-controls { display:grid; grid-template-columns:repeat(5,minmax(0,1fr)); gap:10px; margin:18px 0 22px; }
|
||||
.rep-controls label,.mini-controls label { display:flex; flex-direction:column; justify-content:space-between; gap:8px; min-height:74px; padding:12px; border:1px solid var(--line); background:var(--paper-raised); color:var(--muted); font:.54rem var(--mono); }
|
||||
.rep-controls label > span,.mini-controls label > span { display:flex; justify-content:space-between; gap:8px; }
|
||||
.rep-controls select,.rep-controls input[type="number"],.mini-controls select { width:100%; min-height:31px; border:1px solid var(--line); background:var(--paper); color:var(--ink); font:.57rem var(--mono); }
|
||||
.rep-controls input[type="range"],.mini-controls input[type="range"] { width:100%; accent-color:var(--copper); }
|
||||
.rep-controls output,.mini-controls output { color:var(--copper); }
|
||||
.rep-controls .check-control { flex-direction:row; align-items:center; justify-content:flex-start; }
|
||||
.check-control input { accent-color:var(--copper); }
|
||||
.check-control span { display:flex!important; flex-direction:column; }
|
||||
.check-control b { color:var(--ink); font:600 .59rem var(--sans); }
|
||||
.check-control small { font:.48rem var(--mono); }
|
||||
.token-pipeline { display:grid; grid-template-columns:1fr 24px 1fr 24px 1fr 24px 1fr 24px 1fr; align-items:stretch; margin:24px 0 12px; }
|
||||
.token-pipeline > div { min-height:104px; padding:16px; border:1px solid var(--line); background:var(--paper-raised); }
|
||||
.token-pipeline > div.active { color:#fff; background:var(--navy); }
|
||||
.token-pipeline > i { display:grid; place-items:center; color:var(--copper); font-style:normal; }
|
||||
.token-pipeline span,.rep-metrics span,.position-compare span,.norm-objects span,.route-equation span,.activation-formula span,.model-contrast span,.depth-chart-head span { display:block; color:var(--copper); font:600 .49rem var(--mono); letter-spacing:.1em; }
|
||||
.token-pipeline b { display:block; margin:17px 0 7px; font:650 .7rem var(--serif); }
|
||||
.token-pipeline small { color:var(--muted); font-size:.48rem; line-height:1.5; }
|
||||
.token-pipeline .active small { color:#abb8c6; }
|
||||
.token-chips { display:flex; flex-wrap:wrap; gap:6px; min-height:48px; padding:12px; border:1px solid var(--line); background:rgba(193,124,68,.05); }
|
||||
.token-chips span { display:flex; gap:7px; align-items:center; padding:7px 10px; border:1px solid rgba(193,124,68,.3); background:var(--paper); font:650 .65rem var(--serif); }
|
||||
.token-chips small { color:var(--copper); font:.45rem var(--mono); }
|
||||
.rep-metrics { display:grid; gap:10px; margin:18px 0; }
|
||||
.rep-metrics.four { grid-template-columns:repeat(4,1fr); }
|
||||
.rep-metrics.two { grid-template-columns:repeat(2,1fr); }
|
||||
.rep-metrics article { min-height:112px; padding:16px; border:1px solid var(--line); background:var(--paper-raised); }
|
||||
.rep-metrics article.dark { color:#fff; background:var(--navy); }
|
||||
.rep-metrics b { display:block; margin:15px 0 8px; color:var(--ink); font:650 1.28rem var(--serif); }
|
||||
.rep-metrics .dark b { color:#fff; }
|
||||
.rep-metrics p { margin:0; color:var(--muted); font-size:.51rem; line-height:1.5; }
|
||||
.rep-metrics .dark p { color:#aebac7; }
|
||||
.vector-stage { display:grid; grid-template-columns:.9fr 1.1fr; border:1px solid var(--line-strong); }
|
||||
.vector-copy { padding:24px; background:var(--paper-raised); }
|
||||
.vector-copy > span,.position-readout > span { color:var(--copper); font:600 .52rem var(--mono); letter-spacing:.1em; }
|
||||
.vector-copy h5 { margin:16px 0 10px; font:650 1.05rem var(--serif); }
|
||||
.vector-copy p,.position-readout p,.route-equation p,.activation-formula p { color:var(--muted); font-size:.6rem; line-height:1.65; }
|
||||
.vector-copy dl { display:grid; grid-template-columns:repeat(3,1fr); margin:18px 0 0; }
|
||||
.vector-copy dl div { padding:10px; border-left:1px solid var(--line); }
|
||||
.vector-copy dt { color:var(--muted); font:.46rem var(--mono); }
|
||||
.vector-copy dd { margin:7px 0 0; font:650 .7rem var(--serif); }
|
||||
.vector-stage svg { width:100%; min-height:280px; background:#172437; }
|
||||
.vector-stage svg line,.position-stage svg > line { stroke:#46586d; stroke-width:1; }
|
||||
.vector-stage svg text,.position-stage svg text { fill:#bec8d3; font:11px var(--mono); }
|
||||
.base-point { fill:#d4a176; }.base-line { stroke:#d4a176!important; stroke-width:2!important; }
|
||||
.context-point { fill:#edf1f4; }.context-line { stroke:#edf1f4!important; stroke-width:2.5!important; }
|
||||
.boundary-note { display:grid; grid-template-columns:145px 1fr; gap:18px; align-items:start; margin-top:18px; padding:17px 19px; border-left:3px solid var(--copper); background:rgba(193,124,68,.08); }
|
||||
.boundary-note b { font:650 .72rem var(--serif); }
|
||||
.boundary-note p { margin:0; color:var(--muted); font-size:.58rem; line-height:1.65; }
|
||||
.position-schemes,.norm-topologies,.route-buttons,.activation-buttons { display:grid; grid-template-columns:repeat(4,1fr); gap:8px; }
|
||||
.position-schemes button,.norm-topologies button,.route-buttons button,.activation-buttons button { padding:13px; border:1px solid var(--line); background:var(--paper-raised); text-align:left; cursor:pointer; }
|
||||
.position-schemes button.active,.norm-topologies button.active,.route-buttons button.active,.activation-buttons button.active { color:#fff; border-color:var(--navy); background:var(--navy); }
|
||||
.position-schemes b,.norm-topologies b,.route-buttons b,.activation-buttons b { display:block; font:.58rem var(--mono); }
|
||||
.position-schemes small,.norm-topologies small,.route-buttons small,.activation-buttons small { display:block; margin-top:5px; color:var(--muted); font-size:.47rem; }
|
||||
.position-schemes .active small,.norm-topologies .active small,.route-buttons .active small,.activation-buttons .active small { color:#aebac7; }
|
||||
.position-stage { display:grid; grid-template-columns:1.15fr .85fr; border:1px solid var(--line-strong); background:#172437; }
|
||||
.position-stage svg { width:100%; min-height:330px; }
|
||||
.orbit { fill:none; stroke:#43556a; stroke-dasharray:4 5; }.q-arrow,.k-arrow { stroke-width:3!important; }.q-arrow { stroke:#d69a68!important; }.k-arrow { stroke:#e6ecf2!important; }.q-point { fill:#d69a68; }.k-point { fill:#e6ecf2; }.angle-arc { fill:none; stroke:#8da1b5; stroke-width:2; stroke-dasharray:3 3; }
|
||||
.position-readout { padding:28px; border-left:1px solid #43556a; color:#fff; }
|
||||
.position-readout > b { display:block; margin:24px 0; font:650 1.15rem var(--serif); }
|
||||
.position-readout p { color:#aebac7; }
|
||||
.window-line { position:relative; height:13px; margin:38px 0 10px; background:#34465b; }
|
||||
.window-line i { position:absolute; inset:0 auto 0 0; background:rgba(212,154,104,.45); }
|
||||
.window-line b { position:absolute; top:-5px; width:3px; height:23px; background:#fff; }
|
||||
.position-readout > small { color:#aebac7; font:.5rem var(--mono); }
|
||||
.position-compare,.norm-objects { display:grid; grid-template-columns:repeat(4,1fr); margin-top:16px; }
|
||||
.position-compare article,.norm-objects article { min-height:130px; padding:17px; border:1px solid var(--line); border-right:0; background:var(--paper-raised); }
|
||||
.position-compare article:last-child,.norm-objects article:last-child { border-right:1px solid var(--line); }
|
||||
.position-compare b,.norm-objects b { display:block; margin:18px 0 8px; font:650 .75rem var(--serif); }
|
||||
.position-compare p { margin:0; color:var(--muted); font-size:.52rem; line-height:1.55; }
|
||||
.norm-diagram { padding:23px; border:1px solid var(--line-strong); background:#172437; color:#fff; }
|
||||
.identity-rail { display:grid; grid-template-columns:120px 1fr auto 1fr; align-items:center; gap:12px; }
|
||||
.identity-rail span { color:#d49a68; font:.5rem var(--mono); }
|
||||
.identity-rail i { height:2px; background:#d49a68; }
|
||||
.identity-rail b { font:500 .58rem var(--mono); }
|
||||
.norm-blocks { display:flex; justify-content:center; align-items:center; gap:12px; margin:30px 0 17px; }
|
||||
.norm-blocks div { min-width:150px; padding:17px; border:1px solid #51647a; text-align:center; font:600 .58rem var(--mono); }
|
||||
.norm-blocks .add-node { min-width:48px; border-radius:50%; }
|
||||
.norm-blocks i { color:#d49a68; font-style:normal; }
|
||||
.norm-diagram > p { margin:0; color:#c4ced8; text-align:center; font:.63rem var(--mono); }
|
||||
.depth-chart { margin:17px 0; padding:18px; border:1px solid var(--line); background:var(--paper-raised); }
|
||||
.depth-chart-head { display:flex; justify-content:space-between; align-items:center; }
|
||||
.depth-chart-head b { font:650 1rem var(--serif); }
|
||||
.depth-bars { height:150px; display:flex; align-items:end; gap:5px; margin:16px 0 7px; border-bottom:1px solid var(--line-strong); }
|
||||
.depth-bars i { flex:1; position:relative; min-width:3px; background:linear-gradient(#d49a68,#7c4d2f); }
|
||||
.depth-bars span { position:absolute; top:-16px; font:8px var(--mono); }
|
||||
.depth-axis { display:flex; justify-content:space-between; color:var(--muted); font:.44rem var(--mono); }
|
||||
.norm-objects code { display:block; margin-top:12px; color:var(--muted); font:.53rem var(--mono); white-space:normal; }
|
||||
.residual-grid { display:grid; grid-template-columns:1fr 1fr; gap:14px; }
|
||||
.residual-workbench,.activation-workbench { min-width:0; padding:19px; border:1px solid var(--line-strong); background:var(--paper-raised); }
|
||||
.subhead h5 { margin:0 0 16px; font:650 1.1rem var(--serif); }
|
||||
.route-buttons,.activation-buttons { grid-template-columns:repeat(2,1fr); }
|
||||
.mini-controls { display:grid; grid-template-columns:repeat(2,1fr); gap:8px; margin:12px 0; }
|
||||
.activation-workbench .mini-controls { grid-template-columns:1fr; }
|
||||
.depth-sources { height:185px; display:flex; align-items:end; gap:7px; padding:16px 8px 0; border:1px solid var(--line); background:#172437; }
|
||||
.depth-sources > div { flex:1; height:100%; min-width:20px; display:grid; grid-template-rows:1fr auto auto; gap:4px; align-items:end; text-align:center; }
|
||||
.depth-sources i { width:100%; background:#64798e; }
|
||||
.depth-sources .selected i { background:#d49a68; }
|
||||
.depth-sources b { color:#e7edf2; font:8px var(--mono); }
|
||||
.depth-sources span { color:#93a4b5; font:8px var(--mono); }
|
||||
.route-equation,.activation-formula { min-height:134px; margin-top:12px; padding:16px; border:1px solid var(--line); background:var(--paper); }
|
||||
.route-equation code,.activation-formula code { display:block; margin:13px 0; color:var(--ink); font:.57rem var(--mono); white-space:normal; }
|
||||
.activation-chart { width:100%; height:280px; border:1px solid var(--line); background:#172437; }
|
||||
.activation-chart line { stroke:#43556a; }.activation-chart path { fill:none; stroke:#d49a68; stroke-width:3; }.activation-chart text { fill:#9cafc1; font:10px var(--mono); }
|
||||
.model-contrast { display:grid; grid-template-columns:1fr 54px 1fr; align-items:stretch; margin-top:18px; }
|
||||
.model-contrast article { padding:23px; color:#eef2f5; background:#172437; }
|
||||
.model-contrast article.kimi-card { background:#24384e; }
|
||||
.model-contrast > div { display:grid; place-items:center; color:var(--copper); font:1.2rem var(--serif); }
|
||||
.model-contrast h5 { margin:18px 0 12px; font:650 1rem var(--serif); }
|
||||
.model-contrast ul { margin:0; padding-left:17px; color:#b7c2cd; font-size:.56rem; line-height:1.8; }
|
||||
.rep-lab figcaption { padding:16px 30px; border-top:1px solid var(--line); color:var(--muted); font-size:.54rem; line-height:1.6; }
|
||||
.rep-lab figcaption span { margin-right:10px; color:var(--copper); font:600 .5rem var(--mono); }
|
||||
@media (max-width: 980px) {
|
||||
.rep-lab-head,.rep-panel-intro,.vector-stage,.position-stage,.residual-grid { grid-template-columns:1fr; }
|
||||
.rep-tabs { grid-template-columns:repeat(2,1fr); }
|
||||
.rep-controls { grid-template-columns:repeat(2,1fr); }
|
||||
.token-pipeline { grid-template-columns:1fr; gap:5px; }
|
||||
.token-pipeline > i { transform:rotate(90deg); }
|
||||
.position-readout { border-left:0; border-top:1px solid #43556a; }
|
||||
.rep-metrics.four,.position-compare,.norm-objects { grid-template-columns:repeat(2,1fr); }
|
||||
.model-contrast { grid-template-columns:1fr; }
|
||||
.model-contrast > div { min-height:45px; }
|
||||
}
|
||||
@media (max-width: 560px) {
|
||||
.rep-view,.rep-lab-head { padding:20px 16px; }
|
||||
.rep-tabs { grid-template-columns:1fr; }
|
||||
.rep-tabs button { min-height:75px; border-right:0; border-bottom:1px solid var(--line); }
|
||||
.rep-controls,.mini-controls,.rep-metrics.four,.rep-metrics.two,.position-compare,.norm-objects { grid-template-columns:1fr; }
|
||||
.position-schemes,.norm-topologies { grid-template-columns:repeat(2,1fr); }
|
||||
.route-buttons,.activation-buttons { grid-template-columns:1fr; }
|
||||
.boundary-note { grid-template-columns:1fr; }
|
||||
.vector-copy dl { grid-template-columns:1fr; }
|
||||
.identity-rail { grid-template-columns:1fr; }
|
||||
.identity-rail i { display:none; }
|
||||
.norm-blocks { flex-direction:column; }
|
||||
.norm-blocks i { transform:rotate(90deg); }
|
||||
.depth-sources { gap:3px; padding-inline:4px; }
|
||||
}
|
||||
</style>
|
||||
@@ -10,6 +10,7 @@ const items = [
|
||||
{ id: "k3", href: "/k3/", label: "K3 解剖" },
|
||||
{ id: "deepseek", href: "/deepseek/", label: "DeepSeek" },
|
||||
{ id: "foundations", href: "/foundations/language-models/", label: "基础原理" },
|
||||
{ id: "representation", href: "/architecture/representation/", label: "表示深度" },
|
||||
{ id: "scaling", href: "/scaling/", label: "Scaling" },
|
||||
{ id: "data", href: "/pretraining/data/", label: "数据工程" },
|
||||
{ id: "moe", href: "/moe/", label: "MoE" },
|
||||
|
||||
@@ -60,12 +60,12 @@ export const chapters: Chapter[] = [
|
||||
title: "表示、位置与残差高速公路",
|
||||
kicker: "REPRESENTATION",
|
||||
question: "模型如何知道词序，又如何让信息穿过上百层？",
|
||||
summary: "从分词、位置编码、归一化与激活函数，走到深层网络的信息流和 Attention Residuals。",
|
||||
status: "researching",
|
||||
progress: 18,
|
||||
papers: 16,
|
||||
summary: "用二十张账与四联实验，从 Token/embedding、RoPE/NoPE、Norm 拓扑一路走到 DeepSeek mHC、K3 AttnRes 与 SiTU-GLU。",
|
||||
status: "published",
|
||||
progress: 81,
|
||||
papers: 66,
|
||||
prerequisites: ["02"],
|
||||
highlights: ["RoPE", "RMSNorm", "AttnRes"],
|
||||
highlights: ["二十张表示账", "位置与深度四联实验", "DeepSeek / K3"],
|
||||
},
|
||||
{
|
||||
number: "04",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export type PaperTopic =
|
||||
| "基础"
|
||||
| "表示"
|
||||
| "Transformer"
|
||||
| "长上下文"
|
||||
| "MoE"
|
||||
@@ -3654,10 +3655,252 @@ export const papers: Paper[] = [
|
||||
contribution: "从新近 GitHub issue 持续构建可执行软件任务，以版本化动态集降低仓库级代码评测污染。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2015,
|
||||
title: "Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift",
|
||||
url: "https://arxiv.org/abs/1502.03167",
|
||||
topics: ["表示"],
|
||||
contribution: "以 mini-batch 统计量归一化神经活动，是 LayerNorm 与后续尺度控制路线的重要前史。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2016,
|
||||
title: "Using the Output Embedding to Improve Language Models",
|
||||
url: "https://arxiv.org/abs/1608.05859",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "指出语言模型输出权重矩阵也是有效 embedding，并推荐与输入 embedding 共享参数。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2016,
|
||||
title: "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling",
|
||||
url: "https://arxiv.org/abs/1611.01462",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "从语言模型 loss framework 得到输入词向量与输出分类器权重共享，显著减少词表接口参数。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2018,
|
||||
title: "Deep contextualized word representations",
|
||||
url: "https://arxiv.org/abs/1802.05365",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "ELMo 把词表示写成深双向语言模型内部状态的函数，让同一词的不同 occurrence 随上下文改变。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2020,
|
||||
title: "Transformer Feed-Forward Layers Are Key-Value Memories",
|
||||
url: "https://arxiv.org/abs/2012.14913",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "分析 FFN 第一层方向与输入模式、第二层方向与输出词表分布的关联，揭示逐位置非线性中的模式记忆。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Byte Latent Transformer: Patches Scale Better Than Tokens",
|
||||
url: "https://arxiv.org/abs/2412.09871",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "按 next-byte entropy 形成动态大小 patch，在不使用固定子词词表时按局部复杂度分配主干计算。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2019,
|
||||
title: "Fixup Initialization: Residual Learning Without Normalization",
|
||||
url: "https://arxiv.org/abs/1901.09321",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "以 depth-aware 初始化控制 residual update，使极深残差网络无需归一化也能稳定优化。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2019,
|
||||
title: "Transformers without Tears: Improving the Normalization of Self-Attention",
|
||||
url: "https://arxiv.org/abs/1910.05895",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "系统比较 PreNorm、ScaleNorm 与 FixNorm，展示归一化拓扑、尺度和 embedding 范数的不同作用。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2020,
|
||||
title: "ReZero is All You Need: Fast Convergence at Large Depth",
|
||||
url: "https://arxiv.org/abs/2003.04887",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "给每条 residual branch 加零初始化 scalar gate，使初始网络近似 identity 并支持更深 Transformer。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2020,
|
||||
title: "Understanding the Difficulty of Training Transformers",
|
||||
url: "https://arxiv.org/abs/2004.08249",
|
||||
topics: ["表示", "Transformer", "训练系统"],
|
||||
contribution: "把训练不稳连接到 residual branch 对参数扰动的放大效应，并提出 Admin 自适应初始化。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Hyper-Connections",
|
||||
url: "https://arxiv.org/abs/2409.19606",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "扩宽 residual stream 并学习层输入、流间和层输出映射，为网络深度增加新的连接轴。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2025,
|
||||
title: "mHC: Manifold-Constrained Hyper-Connections",
|
||||
url: "https://arxiv.org/abs/2512.24880",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "把 Hyper-Connections 的 residual mapping 投到特定流形，恢复 identity 性质并改善规模训练稳定性。",
|
||||
spotlight: "DeepSeek",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "nGPT: Normalized Transformer with Representation Learning on the Hypersphere",
|
||||
url: "https://arxiv.org/abs/2410.01131",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "把 embedding、hidden state 与主要权重向量归一化到 hypersphere，把层更新解释为球面位移。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2016,
|
||||
title: "Gaussian Error Linear Units (GELUs)",
|
||||
url: "https://arxiv.org/abs/1606.08415",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "提出平滑的输入相关激活 xΦ(x)，成为 BERT 等 Transformer FFN 的代表非线性。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2017,
|
||||
title: "Searching for Activation Functions",
|
||||
url: "https://arxiv.org/abs/1710.05941",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "通过自动搜索得到 Swish / SiLU 形式 x·sigmoid(βx)，成为 SwiGLU gate 的直接函数前史。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2021,
|
||||
title: "Primer: Searching for Efficient Transformers for Language Modeling",
|
||||
url: "https://arxiv.org/abs/2109.08668",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "在低层算子空间搜索 Transformer，主要收益归于 squared ReLU 与 Q/K/V 后的 depthwise convolution。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2021,
|
||||
title: "ByT5: Towards a Token-Free Future with Pre-trained Byte-to-Byte Models",
|
||||
url: "https://arxiv.org/abs/2105.13626",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "以 UTF-8 bytes 作为模型输入输出单位，系统研究去除固定子词词表的质量与计算折中。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2021,
|
||||
title: "CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation",
|
||||
url: "https://arxiv.org/abs/2103.06874",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "以字符级输入、下采样和深 Transformer 构建不依赖显式 tokenizer 的预训练 encoder。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2021,
|
||||
title: "Charformer: Fast Character Transformers via Gradient-based Subword Tokenization",
|
||||
url: "https://arxiv.org/abs/2106.12672",
|
||||
topics: ["表示", "基础"],
|
||||
contribution: "用可学习的软子词块在字符输入上选择局部组合，探索模型内部自适应 token 边界。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2022,
|
||||
title: "A Length-Extrapolatable Transformer",
|
||||
url: "https://arxiv.org/abs/2212.10554",
|
||||
topics: ["表示", "长上下文"],
|
||||
contribution: "提出 attention resolution 指标与 xPos 等设计，面向训练短、测试长的位置外推。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2023,
|
||||
title: "The Impact of Positional Encoding on Length Generalization in Transformers",
|
||||
url: "https://arxiv.org/abs/2305.19466",
|
||||
topics: ["表示", "长上下文"],
|
||||
contribution: "在特定 decoder-only 推理任务系统比较 APE、T5 RPE、ALiBi、RoPE 与 NoPE，并分析 NoPE 的隐式位置模式。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2023,
|
||||
title: "Functional Interpolation for Relative Positions Improves Long Context Transformers",
|
||||
url: "https://arxiv.org/abs/2310.04418",
|
||||
topics: ["表示", "长上下文"],
|
||||
contribution: "FIRE 以可学习函数表达相对位置并渐进插值，覆盖多种既有 relative bias 形式。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "LongRoPE: Extending LLM Context Window Beyond 2 Million Tokens",
|
||||
url: "https://arxiv.org/abs/2402.13753",
|
||||
topics: ["表示", "长上下文"],
|
||||
contribution: "搜索 RoPE 维度与位置的非均匀插值，并以渐进扩展与短窗恢复延长论文模型上下文。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2020,
|
||||
title: "DeBERTa: Decoding-enhanced BERT with Disentangled Attention",
|
||||
url: "https://arxiv.org/abs/2006.03654",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "在 attention 中分离内容与相对位置向量，并在解码侧增强绝对位置。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2019,
|
||||
title: "BERT Rediscovers the Classical NLP Pipeline",
|
||||
url: "https://arxiv.org/abs/1905.05950",
|
||||
topics: ["表示", "评测"],
|
||||
contribution: "用 edge probing 观察 BERT 不同层中句法与语义信息的可提取性，建立分层表示分析坐标。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2019,
|
||||
title: "Linguistic Knowledge and Transferability of Contextual Representations",
|
||||
url: "https://arxiv.org/abs/1903.08855",
|
||||
topics: ["表示", "评测"],
|
||||
contribution: "跨 ELMo、GPT 与 BERT 的层级线性 probing 比较多类语言知识与迁移特性。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Gemma 2: Improving Open Language Models at a Practical Size",
|
||||
url: "https://arxiv.org/abs/2408.00118",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "组合 local/global attention、pre/post normalization 与 logit soft-capping，展示现代尺度控制配方。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Massive Activations in Large Language Models",
|
||||
url: "https://arxiv.org/abs/2402.17762",
|
||||
topics: ["表示", "低精度"],
|
||||
contribution: "系统观察 LLM 少数 hidden dimensions 中极端大的激活，并分析其跨层、跨 token 行为。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2022,
|
||||
title: "PaLM: Scaling Language Modeling with Pathways",
|
||||
url: "https://arxiv.org/abs/2204.02311",
|
||||
topics: ["表示", "Scaling", "Transformer"],
|
||||
contribution: "在大规模 decoder 中采用 SwiGLU、并行 block 等配方，推动门控 FFN 成为现代主线。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2022,
|
||||
title: "OPT: Open Pre-trained Transformer Language Models",
|
||||
url: "https://arxiv.org/abs/2205.01068",
|
||||
topics: ["表示", "Transformer"],
|
||||
contribution: "公开 Pre-LN decoder 训练配置与模型族，为深度、归一化和复现提供对照。",
|
||||
verified: true,
|
||||
},
|
||||
];
|
||||
|
||||
export const paperTopics: PaperTopic[] = [
|
||||
"基础",
|
||||
"表示",
|
||||
"Transformer",
|
||||
"长上下文",
|
||||
"MoE",
|
||||
|
||||
@@ -0,0 +1,873 @@
|
||||
---
|
||||
import BaseLayout from "@/layouts/BaseLayout.astro";
|
||||
import RepresentationLab from "@/components/RepresentationLab.astro";
|
||||
|
||||
const toc = [
|
||||
["00", "compass", "先拆成二十张账"],
|
||||
["01", "objects", "五种“词向量”"],
|
||||
["02", "token-unit", "Token 是计算单位"],
|
||||
["03", "tying", "输入与输出能否共享"],
|
||||
["04", "contextual", "静态到上下文化"],
|
||||
["05", "ffn-memory", "FFN 也是表示系统"],
|
||||
["06", "history", "八次历史转向"],
|
||||
["07", "symmetry", "位置先打破什么对称"],
|
||||
["08", "absolute", "绝对位置"],
|
||||
["09", "relative", "相对位置"],
|
||||
["10", "rope", "RoPE 的二维几何"],
|
||||
["11", "extrapolation", "长度外推为何会坏"],
|
||||
["12", "nope", "NoPE 不等于无顺序"],
|
||||
["13", "deepseek-rope", "DeepSeek 位置谱系"],
|
||||
["14", "norm-object", "Norm 控制谁"],
|
||||
["15", "norm-topology", "Pre / Post 是拓扑"],
|
||||
["16", "deep-stack", "堆深的稳定谱系"],
|
||||
["17", "residual", "Residual 的成就与盲点"],
|
||||
["18", "hc", "Hyper-Connections"],
|
||||
["19", "mhc", "mHC 的流形约束"],
|
||||
["20", "attnres", "AttnRes 沿深度检索"],
|
||||
["21", "k3-block", "K3 的 9 个深度来源"],
|
||||
["22", "activation", "激活函数谱系"],
|
||||
["23", "outliers", "从 clamp 到 SiTU"],
|
||||
["24", "deepseek", "DeepSeek 重点复盘"],
|
||||
["25", "k3", "K3 四轴合流"],
|
||||
["26", "lab", "四联交互工作台"],
|
||||
["27", "audit", "读架构图的审计卡"],
|
||||
["↳", "papers", "66 个关键节点"],
|
||||
];
|
||||
|
||||
const ledgers = [
|
||||
["Q01 / UNIT", "单位账", "字符、byte、subword、token、patch 谁是模型一步？", "边界先决定序列长度、词表和计算密度。"],
|
||||
["Q02 / PARAM", "参数账", "词表变大，哪张矩阵跟着变？", "输入 embedding 和输出 head 常各含 V×d 参数。"],
|
||||
["Q03 / STATIC", "静态表示账", "同一个 token ID 是否永远同义？", "第 0 层查表固定，深层 hidden state 随上下文变化。"],
|
||||
["Q04 / CONTEXT", "上下文化账", "多义性在哪里被展开？", "序列混合与逐位置 FFN 反复改写每次 occurrence。"],
|
||||
["Q05 / OUTPUT", "输出几何账", "hidden state 怎样变成下一个 token？", "与输出向量点积成 logits，再经 softmax。"],
|
||||
["Q06 / SYMMETRY", "对称性账", "无位置 Attention 知道先后吗？", "裸算子排列等变；causal mask 只给有向可见性。"],
|
||||
["Q07 / ABSOLUTE", "绝对位置账", "“第 37 位”如何进入表示？", "位置向量可加到内容，但两者从入口便纠缠。"],
|
||||
["Q08 / RELATIVE", "相对位置账", "“相隔 3”如何进入匹配？", "距离表示或 bias 可直接改变位置对的 score。"],
|
||||
["Q09 / ROTATION", "旋转账", "RoPE 为什么要成对旋转维度？", "两次绝对旋转在 Q·K 中相消为相对旋转。"],
|
||||
["Q10 / LENGTH", "外推账", "公式能算更长，为何模型仍可能坏？", "新相位、距离、bias 与注意力分辨率可能越出训练分布。"],
|
||||
["Q11 / NOPE", "NoPE 账", "无显式位置是否等于无顺序？", "因果边界、递归衰减和内容统计都可能隐式带序。"],
|
||||
["Q12 / TARGET", "统计账", "LayerNorm、RMSNorm、QK-Norm在管谁？", "它们控制不同对象，不能互相替换名字。"],
|
||||
["Q13 / TOPOLOGY", "拓扑账", "Norm 在分支前还是相加后？", "位置改变 identity path、初始化梯度与深度动态。"],
|
||||
["Q14 / SCALE", "幅值账", "有 Residual 为何 state 仍会长大？", "固定单位累加没有自动控制总尺度。"],
|
||||
["Q15 / GRADIENT", "梯度账", "有捷径为何仍会训练不稳？", "可达性不等于各层更新和 attention logits 都平衡。"],
|
||||
["Q16 / DILUTION", "稀释账", "早期层为何会逐渐听不见？", "总和持续增长时，单层增量的相对份额下降。"],
|
||||
["Q17 / WIDTH", "拓宽账", "Residual stream 必须只有一条吗？", "HC 扩宽成多条流，学习跨层连接。"],
|
||||
["Q18 / SELECT", "选择账", "深度能否像序列一样 Attention？", "AttnRes 对历史层或块做内容相关 softmax。"],
|
||||
["Q19 / FFN", "FFN 账", "Attention 后为何还要大 MLP？", "前者跨位置搬运，后者逐位置做非线性加工。"],
|
||||
["Q20 / OUTLIER", "极值账", "乘法门控为何也会爆？", "SwiGLU 两支可无界；clamp 与 SiTU 以不同方式限幅。"],
|
||||
];
|
||||
|
||||
const waves = [
|
||||
["2003–13", "从 one-hot 到连续几何", "NPLM · word2vec", "相似上下文可共享统计，但一个词仍只有一个静态点。"],
|
||||
["2015–18", "开放词表与上下文化", "BPE · SentencePiece · ELMo · BERT", "先解决边界，再让每次出现拥有不同状态。"],
|
||||
["2015–17", "深层捷径与单样本归一化", "ResNet · LayerNorm · Transformer", "网络先能堆起来，固定累加的幅值问题随后出现。"],
|
||||
["2018–21", "把顺序写进 Attention", "Relative PE · Transformer-XL · RoPE · ALiBi", "从绝对入口向 pair-wise 距离与旋转几何转移。"],
|
||||
["2019–22", "稳定训练更深 Transformer", "RMSNorm · Pre-LN · ReZero · Admin · DeepNorm", "优化稳定、表示变化与最终质量并非同一目标。"],
|
||||
["2021–24", "跨出训练窗口", "xPos · NoPE · PI · YaRN · FIRE · LongRoPE", "位置外推变成独立问题，但不替代长程数据和检索。"],
|
||||
["2024–26", "重写 residual stream", "HC · nGPT · mHC · AttnRes", "一条固定总线变成多流映射或沿深度选择。"],
|
||||
["2024–26", "前沿模型协同设计", "DeepSeek-V2/V4 · Kimi Linear/K3", "位置、Norm、Residual 与激活开始按缓存、低精度和系统共同设计。"],
|
||||
];
|
||||
|
||||
const representationObjects = [
|
||||
["01", "TOKEN ID", "离散索引", "它只说“查哪一行”，没有距离、方向或概率。"],
|
||||
["02", "INPUT EMBEDDING", "E[id]", "同一 ID 的第 0 层查表结果；还没有读到句内上下文。"],
|
||||
["03", "HIDDEN STATE", "hᶫₜ", "同一 ID 在不同句子、位置和层中通常不同。"],
|
||||
["04", "OUTPUT VECTOR", "Wout[v]", "与最终 state 点积，扮演词表 classifier 的类别向量。"],
|
||||
["05", "PROBABILITY", "softmax(logits)", "整个上下文计算后的分布，不是某个向量独自携带的属性。"],
|
||||
];
|
||||
|
||||
const positionRows = [
|
||||
["Absolute add", "xₜ = eₜ + pₜ", "入口 state", "直接给绝对坐标；内容与位置从入口混合。"],
|
||||
["Relative repr.", "scoreᵢⱼ += qᵢᵀaᵢ₋ⱼ", "Q–K pair", "直接表达距离；可做 buckets 或截断。"],
|
||||
["RoPE", "qᵢᵀRⱼ₋ᵢkⱼ", "Q / K 旋转", "绝对角度进入向量，相对角度留在内积。"],
|
||||
["ALiBi", "scoreᵢⱼ − mₕ|i−j|", "attention logit", "不给 embedding 加位置，只施加 head-wise 近邻偏置。"],
|
||||
["NoPE", "explicit term = 0", "无显式插槽", "必须继续问 causal mask、递归状态和数据怎样提供顺序。"],
|
||||
];
|
||||
|
||||
const normRows = [
|
||||
["LayerNorm", "单 token 的全部特征", "去均值 + 除标准差", "re-centering + re-scaling"],
|
||||
["RMSNorm", "单 token 的全部特征", "只除 RMS", "re-scaling"],
|
||||
["QK-Norm", "每个 attention head 的 Q / K", "先归一化再学习温度", "防 logit 任意饱和"],
|
||||
["Latent Norm", "压缩后的 cKV / routed latent", "在低维状态上控制尺度", "不是 block PreNorm"],
|
||||
["AttnRes key Norm", "不同深度来源的 keys", "比较前先做 RMSNorm", "避免大幅值层靠尺度胜出"],
|
||||
];
|
||||
|
||||
const deepStack = [
|
||||
["2015", "ResNet", "identity shortcut", "先解决深网优化退化；没有给每层贡献可学习权重。"],
|
||||
["2019", "Fixup", "depth-scaled init", "用初始化控制初始 residual update，不依赖 Norm。"],
|
||||
["2019", "RMSNorm", "scale only", "去掉 re-centering，简化归一化。"],
|
||||
["2020", "Pre-LN analysis", "Norm before branch", "裸 identity path 改善初始化梯度，但会固定累加所有分支。"],
|
||||
["2020", "ReZero / Admin", "gate / adaptive init", "从近 identity 开始，或控制 residual 对参数扰动的放大。"],
|
||||
["2021", "NormFormer", "extra internal Norms", "修补 Pre-LN 深浅层梯度尺度不匹配。"],
|
||||
["2022", "DeepNorm", "scaled Post-LN", "用 residual scale 与初始化稳定到 1,000 层。"],
|
||||
["2024–26", "HC → mHC → AttnRes", "new depth topology", "从控制单位加法，走向多流连接或选择性深度读取。"],
|
||||
];
|
||||
|
||||
const activationRows = [
|
||||
["ReLU", "max(0,x)", "单支、分段线性", "正向无界；负半轴为零"],
|
||||
["GELU", "x Φ(x)", "单支、平滑", "正向无界；负向平滑衰减"],
|
||||
["GLU", "g ⊙ σ(u)", "内容 × 有界 gate", "内容支仍无界"],
|
||||
["SwiGLU", "Swish(g) ⊙ u", "两条可学习支相乘", "两个乘法因子都可无界"],
|
||||
["V4 clamp", "cap(Swish(g)) ⊙ clamp(u)", "硬限幅", "作者配方：linear [−10,10]，gate 上界 10"],
|
||||
["K3 SiTU", "4tanh(g/4)σ(g) · 25tanh(u/25)", "平滑限幅", "逐点 |y| < 100"],
|
||||
];
|
||||
|
||||
const deepseekRows = [
|
||||
["DeepSeek LLM / 2024", "Pre-Norm RMSNorm · SwiGLU · RoPE", "建立现代 dense 基线；Norm、激活与位置是三条独立轴。"],
|
||||
["DeepSeek-V2 / 2024", "MLA · decoupled RoPE", "内容 K/V 可低秩压缩并吸收投影；位置另走 shared key / multi-head query。"],
|
||||
["DeepSeek-V3 / 2024", "MLA · extra latent RMSNorm", "除 block PreNorm 外，再控制 compressed latent；不要把两个 Norm 混为一处。"],
|
||||
["DeepSeek-V3.2 / 2025", "稀疏索引继续接入 MLA 谱系", "位置、内容与索引需要在高效注意力里共同保真。"],
|
||||
["mHC / 2025", "4× residual streams · Birkhoff constraint", "把 HC 的自由映射投到 doubly stochastic manifold。"],
|
||||
["DeepSeek-V4 / 2026", "mHC · partial RoPE · Q/KV RMSNorm · SwiGLU clamp", "最后 64 维承载旋转；残差、匹配尺度和 FFN 极值一起控制。"],
|
||||
];
|
||||
|
||||
const auditRows = [
|
||||
["单位", "原始字符串怎样分成模型一步？训练和推理使用同一 tokenizer / patcher 吗？"],
|
||||
["接口参数", "V、d、input embedding、output head 是否 tying？参数和显存怎样算？"],
|
||||
["状态", "图里画的是初始 embedding、某层 hidden state、KV latent 还是 logits？"],
|
||||
["位置入口", "位置加在 embedding、Q/K、logit、Value、递归 gate 还是完全隐式？"],
|
||||
["相对性", "“relative”来自公式恒等式、learned bias、bucket 还是模型经验模式？"],
|
||||
["外推", "训练长度、微调长度、测试长度和真实利用长度是否分开报告？"],
|
||||
["Norm 对象", "state、Q/K、latent、expert aggregate、depth key 中哪一个被归一化？"],
|
||||
["Norm 拓扑", "Pre、Post、sandwich、parallel 或额外 latent Norm 放在计算图哪里？"],
|
||||
["Residual", "固定单位相加、可学习 scalar、多流矩阵还是沿深度 attention？"],
|
||||
["状态成本", "需要保留 1 条 stream、n 条 stream、所有层还是 block summaries？"],
|
||||
["激活", "单支还是门控双支？是否有数学上界、硬 clamp 或只是一项经验稳定措施？"],
|
||||
["证据", "结论是函数性质、论文理论、作者自报实验、第三方复跑还是本站教学推导？"],
|
||||
];
|
||||
|
||||
const paperChain = [
|
||||
["2003","A Neural Probabilistic Language Model","https://www.jmlr.org/papers/v3/bengio03a.html","离散词查表进入共同学习的连续表示。"],
|
||||
["2013","Efficient Estimation of Word Representations","https://arxiv.org/abs/1301.3781","以高效目标大规模学习静态词向量。"],
|
||||
["2015","Batch Normalization","https://arxiv.org/abs/1502.03167","归一化神经活动的 batch 统计前史。"],
|
||||
["2015","Deep Residual Learning","https://arxiv.org/abs/1512.03385","identity shortcut 把深度优化改写为 residual learning。"],
|
||||
["2015","BPE for Neural Machine Translation","https://arxiv.org/abs/1508.07909","以子词处理稀有词和开放词表。"],
|
||||
["2016","Layer Normalization","https://arxiv.org/abs/1607.06450","单样本内按整层特征统计归一化。"],
|
||||
["2016","Using the Output Embedding","https://arxiv.org/abs/1608.05859","输出 classifier 也是 embedding，并可与输入表共享。"],
|
||||
["2016","Tying Word Vectors and Word Classifiers","https://arxiv.org/abs/1611.01462","从 loss framework 得到 weight tying。"],
|
||||
["2016","Gaussian Error Linear Units","https://arxiv.org/abs/1606.08415","以输入大小做平滑、概率式加权。"],
|
||||
["2016","Language Modeling with Gated ConvNets","https://arxiv.org/abs/1612.08083","GLU 用内容分支乘 sigmoid gate。"],
|
||||
["2017","Attention Is All You Need","https://proceedings.neurips.cc/paper/7181-attention-is-all-you-need","sinusoid、Post-LN residual 与逐位置 FFN 的原始组合。"],
|
||||
["2017","Searching for Activation Functions","https://arxiv.org/abs/1710.05941","搜索得到 Swish：x·sigmoid(βx)。"],
|
||||
["2018","SentencePiece","https://arxiv.org/abs/1808.06226","从原始句子训练语言无关子词模型。"],
|
||||
["2018","Deep Contextualized Word Representations","https://arxiv.org/abs/1802.05365","同一词的表示变成深双向 LM 状态的函数。"],
|
||||
["2018","Self-Attention with Relative Position","https://aclanthology.org/N18-2074/","把相对距离表示加入 attention 的 key/value 路径。"],
|
||||
["2018","BERT","https://arxiv.org/abs/1810.04805","深双向 Transformer 上下文化表示成为迁移接口。"],
|
||||
["2019","Transformer-XL","https://aclanthology.org/P19-1285/","跨段 recurrence 与相对位置共同延长依赖。"],
|
||||
["2019","Fixup Initialization","https://arxiv.org/abs/1901.09321","用初始化缩放训练无归一化 residual networks。"],
|
||||
["2019","RMSNorm","https://arxiv.org/abs/1910.07467","去掉 re-centering，只保留 RMS re-scaling。"],
|
||||
["2019","T5","https://jmlr.org/papers/v21/20-074.html","以 bucketed relative bias 进入统一 text-to-text 系统。"],
|
||||
["2019","Transformers without Tears","https://arxiv.org/abs/1910.05895","PreNorm、ScaleNorm 与 FixNorm 的系统比较。"],
|
||||
["2020","On Layer Normalization in Transformers","https://proceedings.mlr.press/v119/xiong20b.html","分析 Post-LN / Pre-LN 的初始化梯度与 warm-up。"],
|
||||
["2020","GLU Variants Improve Transformer","https://arxiv.org/abs/2002.05202","GEGLU / SwiGLU 等门控 FFN 变体。"],
|
||||
["2020","ReZero","https://arxiv.org/abs/2003.04887","零初始化 residual gate 让网络从 identity 开始。"],
|
||||
["2020","Understanding Training Difficulty / Admin","https://arxiv.org/abs/2004.08249","把不稳连接到 residual 对参数扰动的放大。"],
|
||||
["2020","Query-Key Normalization","https://arxiv.org/abs/2010.04245","直接控制 Q/K 尺度和 softmax 饱和。"],
|
||||
["2020","FFN Layers Are Key-Value Memories","https://arxiv.org/abs/2012.14913","分析 FFN 方向与输入模式、输出词表的关联。"],
|
||||
["2021","RoFormer / RoPE","https://arxiv.org/abs/2104.09864","绝对旋转在 Q·K 中形成相对位置。"],
|
||||
["2021","ALiBi","https://arxiv.org/abs/2108.12409","直接给 attention logits 添加距离惩罚。"],
|
||||
["2021","NormFormer","https://arxiv.org/abs/2110.09456","以额外内部 Norm 修补 Pre-LN 梯度尺度。"],
|
||||
["2021","Primer","https://arxiv.org/abs/2109.08668","搜索得到 squared ReLU 与 depthwise QKV conv。"],
|
||||
["2021","ByT5","https://arxiv.org/abs/2105.13626","无固定子词词表的 byte-to-byte 预训练。"],
|
||||
["2021","CANINE","https://arxiv.org/abs/2103.06874","字符级 encoder 避免显式 tokenizer。"],
|
||||
["2021","Charformer","https://arxiv.org/abs/2106.12672","学习软子词块的 token-free 路线。"],
|
||||
["2022","DeepNet / DeepNorm","https://arxiv.org/abs/2203.00555","residual scale 与初始化把 Transformer 堆到 1,000 层。"],
|
||||
["2022","A Length-Extrapolatable Transformer","https://arxiv.org/abs/2212.10554","以 attention resolution 与 xPos 改善长度外推。"],
|
||||
["2023","LLaMA","https://arxiv.org/abs/2302.13971","RMSNorm、SwiGLU、RoPE 的代表性现代配方。"],
|
||||
["2023","Impact of Positional Encoding","https://arxiv.org/abs/2305.19466","在特定推理任务系统比较 APE、RPE、RoPE、ALiBi、NoPE。"],
|
||||
["2023","Position Interpolation","https://arxiv.org/abs/2306.15595","把扩展位置映回训练过的 RoPE 区间。"],
|
||||
["2023","YaRN","https://arxiv.org/abs/2309.00071","分频率插值并调节 attention 温度。"],
|
||||
["2023","FIRE","https://arxiv.org/abs/2310.04418","学习 relative position 函数并渐进插值。"],
|
||||
["2024","LongRoPE","https://arxiv.org/abs/2402.13753","搜索非均匀插值并渐进扩展到论文设定的 2,048K。"],
|
||||
["2024","DeepSeek LLM","https://arxiv.org/abs/2401.02954","Pre-Norm RMSNorm、SwiGLU 与 RoPE 的 DeepSeek 基线。"],
|
||||
["2024","DeepSeek-V2","https://arxiv.org/abs/2405.04434","MLA 以 decoupled RoPE 分离内容压缩与位置。"],
|
||||
["2024","DeepSeek-V3","https://arxiv.org/abs/2412.19437","在 compressed latent 后增加 RMSNorm。"],
|
||||
["2024","Hyper-Connections","https://arxiv.org/abs/2409.19606","扩宽 residual stream 并动态学习层间映射。"],
|
||||
["2024","nGPT","https://arxiv.org/abs/2410.01131","让表示、hidden state 与权重在 hypersphere 上学习。"],
|
||||
["2024","Byte Latent Transformer","https://arxiv.org/abs/2412.09871","按 next-byte entropy 形成动态 patches。"],
|
||||
["2025","Kimi Linear","https://arxiv.org/abs/2510.26692","KDA 隐式位置与 MLA NoPE 的 3:1 混合。"],
|
||||
["2025","DeepSeek-V3.2","https://arxiv.org/abs/2512.02556","稀疏注意力继续接入 MLA 与长上下文谱系。"],
|
||||
["2025","mHC","https://arxiv.org/abs/2512.24880","把 HC residual mapping 投到受约束流形。"],
|
||||
["2026","Attention Residuals","https://arxiv.org/abs/2603.15031","对先前层或 block 表示做内容相关深度聚合。"],
|
||||
["2026","DeepSeek-V4","https://arxiv.org/abs/2606.19348","mHC、partial RoPE、Q/KV Norm 与 clamp 合流。"],
|
||||
["2026","Kimi K3","https://arxiv.org/abs/2607.24653","NoPE hybrid、Block AttnRes 与 SiTU-GLU 合流。"],
|
||||
["2018","GPT-1","https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf","causal decoder hidden state 成为迁移表示。"],
|
||||
["2019","BERT Rediscovers the NLP Pipeline","https://arxiv.org/abs/1905.05950","用 probes 观察语言信息在层间的分布。"],
|
||||
["2019","Linguistic Knowledge in Contextual Representations","https://arxiv.org/abs/1903.08855","跨 ELMo、GPT、BERT 做分层线性 probing。"],
|
||||
["2020","DeBERTa","https://arxiv.org/abs/2006.03654","把内容与位置表示解耦进 attention。"],
|
||||
["2022","PaLM","https://arxiv.org/abs/2204.02311","大规模 decoder 采用 SwiGLU 等配方。"],
|
||||
["2022","OPT","https://arxiv.org/abs/2205.01068","公开 Pre-LN decoder 的规模复现坐标。"],
|
||||
["2023","LLaMA 2","https://arxiv.org/abs/2307.09288","延续 RoPE、RMSNorm 与 SwiGLU 配方。"],
|
||||
["2023","Small-Scale Proxies for Training Instabilities","https://arxiv.org/abs/2309.14322","以 QK-LayerNorm 等干预研究规模训练不稳。"],
|
||||
["2024","Gemma 2","https://arxiv.org/abs/2408.00118","组合 pre/post normalization 与 logit soft-capping。"],
|
||||
["2024","Massive Activations in LLMs","https://arxiv.org/abs/2402.17762","系统观察少量异常大的 hidden activations。"],
|
||||
["2025","Kimi K2","https://arxiv.org/abs/2507.20534","K3 之前的 MLA、RMSNorm 与稳定训练配方。"],
|
||||
["2026","Stable LatentMoE / K3 §2.3","https://arxiv.org/abs/2607.24653","把 routed latent Norm 与有界 SiTU-GLU 放进专家路径。"],
|
||||
];
|
||||
---
|
||||
|
||||
<BaseLayout
|
||||
title="表示、位置与残差高速公路：从 Token 向量到 Kimi K3"
|
||||
description="用二十张问题账和四联交互实验，系统理解 embedding、上下文化表示、位置编码、RoPE/NoPE、LayerNorm/RMSNorm、Pre/Post-LN、mHC、Attention Residuals、SwiGLU 与 Kimi K3 SiTU-GLU。"
|
||||
section="representation"
|
||||
>
|
||||
<section class="chapter-hero">
|
||||
<div class="chapter-hero-main">
|
||||
<p class="eyebrow"><span>CHAPTER 03</span> REPRESENTATION · POSITION · DEPTH</p>
|
||||
<h1>表示、位置与<br /><em>残差高速公路</em></h1>
|
||||
<p class="chapter-deck">
|
||||
Token 不是词，embedding 不是知识，位置也不是一串神秘正弦。真正的问题是：
|
||||
一条离散符号怎样获得上下文、顺序和深度，又怎样在 93 层与乘法门控中不被放大或稀释？
|
||||
</p>
|
||||
<div class="hero-thesis">
|
||||
<span>ONE SENTENCE</span>
|
||||
<p>模型的 hidden state 同时沿 <b>词表接口、序列位置、网络深度、局部非线性</b> 四个坐标轴被改写；K3 正是把四轴分别重做后再拼成一套系统。</p>
|
||||
</div>
|
||||
</div>
|
||||
<aside class="chapter-hero-aside">
|
||||
<span>READING CONTRACT</span>
|
||||
<strong>20 + 8 + 66</strong>
|
||||
<p>二十张问题账 · 八次历史转向 · 六十六个一手节点</p>
|
||||
<dl>
|
||||
<div><dt>DIFFICULTY</dt><dd>入门 → 论文级</dd></div>
|
||||
<div><dt>LABS</dt><dd>4 个可操作实验</dd></div>
|
||||
<div><dt>FOCUS</dt><dd>DeepSeek / Kimi</dd></div>
|
||||
<div><dt>EVIDENCE</dt><dd>原文 / 推导分层</dd></div>
|
||||
</dl>
|
||||
</aside>
|
||||
</section>
|
||||
|
||||
<div class="article-shell">
|
||||
<aside class="side-rail">
|
||||
<span>CONTENTS</span>
|
||||
<nav aria-label="本章目录">
|
||||
{toc.map(([number, id, label]) => <a href={`#${id}`}><b>{number}</b>{label}</a>)}
|
||||
</nav>
|
||||
</aside>
|
||||
|
||||
<article class="article">
|
||||
<section class="article-section opening-section" id="compass">
|
||||
<p class="section-kicker">00 / PROBLEM LEDGERS</p>
|
||||
<h2>先别背 RoPE：把一条 state 拆成二十本账</h2>
|
||||
<p class="lead">
|
||||
这章最危险的读法，是把 Tokenizer、位置编码、Norm、Residual 和激活函数当作五个互不相干的组件。
|
||||
它们其实共同决定一件事：<strong>同一条向量在进入模型、匹配上下文、穿过深度、通过非线性时，哪些信息和尺度能被保留。</strong>
|
||||
</p>
|
||||
<div class="rep-ledgers">
|
||||
{ledgers.map(([code, title, question, answer]) => (
|
||||
<article>
|
||||
<span>{code}</span><h3>{title}</h3><p>{question}</p><b>{answer}</b>
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
<div class="four-axis">
|
||||
<div><span>VOCABULARY AXIS</span><b>ID → vector → logits</b><p>谁是模型的一步，入口和出口各花多少参数？</p></div>
|
||||
<div><span>SEQUENCE AXIS</span><b>content + order</b><p>位置怎样改变 token-to-token 匹配？</p></div>
|
||||
<div><span>DEPTH AXIS</span><b>layer 0 → layer L</b><p>捷径怎样避免阻塞，又怎样避免所有层固定混成一团？</p></div>
|
||||
<div><span>NONLINEAR AXIS</span><b>FFN / GLU / experts</b><p>逐位置加工怎样增加容量，又怎样制造极端激活？</p></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="objects">
|
||||
<p class="section-kicker">01 / REPRESENTATION OBJECTS</p>
|
||||
<h2>“词向量”这个词，至少藏着五个不同对象</h2>
|
||||
<p class="lead">如果不先区分对象，后面所有讨论都会滑坡：把 ID 当语义、把 embedding 当知识、把概率当一个向量固有属性。</p>
|
||||
<div class="object-stack">
|
||||
{representationObjects.map(([number, name, formula, note]) => (
|
||||
<article><span>{number}</span><div><b>{name}</b><code>{formula}</code></div><p>{note}</p></article>
|
||||
))}
|
||||
</div>
|
||||
<div class="formula-panel">
|
||||
<span>MINIMUM PIPELINE</span>
|
||||
<code>t → eₜ = E[t] → hₜ⁰ → hₜ¹ → … → hₜᴸ → zᵥ = W<sub>out</sub>[v]ᵀhₜᴸ → softmax(z)</code>
|
||||
<p>同一个 token ID 的 `E[t]` 固定；真正随上下文变化的是层内状态 `hᶫₜ`。输出概率还依赖整个词表的相对 logits。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="token-unit">
|
||||
<p class="section-kicker">02 / TOKEN AS COMPUTE UNIT</p>
|
||||
<h2>Tokenizer 不是预处理小工具，它先写下三笔架构账</h2>
|
||||
<div class="tradeoff-triangle">
|
||||
<article><span>VOCABULARY</span><b>V 大</b><p>embedding / output head 参数增大，常见词可用更少步表示。</p></article>
|
||||
<article><span>SEQUENCE</span><b>T 长</b><p>Attention、状态更新、训练 token budget 与推理步数都增加。</p></article>
|
||||
<article><span>BOUNDARY</span><b>切在哪里</b><p>语言、数字、代码、稀有词和拼写鲁棒性会获得不同归纳偏置。</p></article>
|
||||
</div>
|
||||
<div class="unit-ladder">
|
||||
<div><span>WORD</span><b>少步 / 大词表 / OOV</b><i></i></div>
|
||||
<div><span>SUBWORD</span><b>固定折中 / 现代主流</b><i></i></div>
|
||||
<div><span>CHAR / BYTE</span><b>小词表 / 长序列</b><i></i></div>
|
||||
<div><span>DYNAMIC PATCH</span><b>按局部熵分配步数</b></div>
|
||||
</div>
|
||||
<div class="evidence-note">
|
||||
<span>BLT / 2024</span>
|
||||
<p>Byte Latent Transformer 不把每个 byte 都交给大 Transformer；它按 next-byte entropy 形成动态 patch，在难预测位置分配更多计算。论文结果属于其 FLOP-controlled scaling 设置，不是“固定 tokenizer 已被淘汰”。</p>
|
||||
<a href="https://arxiv.org/abs/2412.09871">阅读原文 →</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="tying">
|
||||
<p class="section-kicker">03 / INPUT–OUTPUT GEOMETRY</p>
|
||||
<h2>模型从一张表里“读入”，也可以用同一张表“分类输出”</h2>
|
||||
<div class="tying-diagram">
|
||||
<div><span>INPUT</span><b>E[t]</b><small>按 ID 查一行</small></div>
|
||||
<i>→</i><div class="backbone-box"><span>BACKBONE</span><b>context → h</b><small>反复改写 state</small></div>
|
||||
<i>→</i><div><span>OUTPUT</span><b>E[v]ᵀh</b><small>与每个候选行点积</small></div>
|
||||
<svg viewBox="0 0 760 110" aria-hidden="true"><path d="M650 20 C650 90 110 90 110 20" /><text x="330" y="98">WEIGHT TYING</text></svg>
|
||||
</div>
|
||||
<div class="comparison-table">
|
||||
<div class="table-row head"><span>设计</span><span>接口参数</span><span>直觉</span><span>边界</span></div>
|
||||
<div class="table-row"><b>Untied</b><span>2Vd</span><span>输入与输出各学一套几何</span><span>更自由，也更昂贵</span></div>
|
||||
<div class="table-row"><b>Tied</b><span>Vd</span><span>读入和分类共享类别向量</span><span>角色仍不同，不等于计算相同</span></div>
|
||||
</div>
|
||||
<p class="source-line">Press & Wolf 与 Inan et al. 在 2016 年独立把 weight tying 推到语言模型主线：<a href="https://arxiv.org/abs/1608.05859">Output Embedding</a> · <a href="https://arxiv.org/abs/1611.01462">Tying Word Vectors</a></p>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="contextual">
|
||||
<p class="section-kicker">04 / CONTEXTUALIZATION</p>
|
||||
<h2>同一个 token 的“词典坐标”，怎样变成这句话里的状态？</h2>
|
||||
<div class="context-story">
|
||||
<article><span>LAYER 0</span><h3>固定查表</h3><p>“bank” 无论在河岸还是金融句中先取同一行。</p><b>type-level</b></article>
|
||||
<i>→</i>
|
||||
<article><span>LOWER LAYERS</span><h3>局部与形式</h3><p>邻近词、词形、标点与短程组合开始改变方向。</p><b>occurrence-level</b></article>
|
||||
<i>→</i>
|
||||
<article><span>MIDDLE / UPPER</span><h3>句内任务状态</h3><p>与语义、指代、预测目标相关的特征继续被重组。</p><b>context-conditioned</b></article>
|
||||
<i>→</i>
|
||||
<article><span>LM HEAD</span><h3>候选分布</h3><p>最终 state 与全部输出向量比较，产生下一 token 分布。</p><b>decision interface</b></article>
|
||||
</div>
|
||||
<blockquote>
|
||||
<b>不要把 probe 读成“解剖标签”。</b>
|
||||
ELMo、BERT 和后续 probing 工作展示不同层可线性提取不同语言信息；它们没有证明某层只负责一种功能，也没有证明 probe 读出的关联就是模型决策的因果电路。
|
||||
</blockquote>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="ffn-memory">
|
||||
<p class="section-kicker">05 / FFN AS REPRESENTATION</p>
|
||||
<h2>Attention 负责“从哪里搬”，FFN 负责“在当前位置怎样改写”</h2>
|
||||
<div class="attention-ffn">
|
||||
<article>
|
||||
<span>CROSS-TOKEN</span><h3>Attention</h3>
|
||||
<div class="token-row"><i>A</i><i>B</i><i>C</i><i>D</i></div>
|
||||
<p>每个位置按 Q/K 权重汇总其他位置的 Value；它重排和混合序列信息。</p>
|
||||
</article>
|
||||
<div aria-hidden="true">＋</div>
|
||||
<article>
|
||||
<span>WITHIN-TOKEN</span><h3>FFN / GLU / MoE</h3>
|
||||
<div class="ffn-glyph"><i>d</i><b>→</b><i>≈4d</i><b>→</b><i>d</i></div>
|
||||
<p>同一组权重逐位置应用，产生非线性特征、门控与大量参数容量。</p>
|
||||
</article>
|
||||
</div>
|
||||
<p>
|
||||
Geva et al. 把 FFN 分析为 key-value memories：第一层方向与输入模式相关，第二层方向诱导输出词表分布；
|
||||
低层更多浅层模式，高层更语义化。这个视角说明 FFN 不是“Attention 后的填充”，但也不能把每个 neuron 直接命名成一条稳定事实。
|
||||
</p>
|
||||
<p class="source-line"><a href="https://arxiv.org/abs/2012.14913">Transformer Feed-Forward Layers Are Key-Value Memories →</a></p>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="history">
|
||||
<p class="section-kicker">06 / EIGHT TURNS</p>
|
||||
<h2>三十年不是一条“更复杂”的直线，而是八次瓶颈迁移</h2>
|
||||
<div class="wave-timeline">
|
||||
{waves.map(([years, title, papers, note], index) => (
|
||||
<article><span>{String(index + 1).padStart(2,"0")} / {years}</span><h3>{title}</h3><b>{papers}</b><p>{note}</p></article>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="symmetry">
|
||||
<p class="section-kicker">07 / PERMUTATION SYMMETRY</p>
|
||||
<h2>Attention 先天会匹配内容，却先天不知道“谁在前面”</h2>
|
||||
<div class="symmetry-proof">
|
||||
<div><span>ORIGINAL</span><b>[猫, 追, 狗]</b><p>每行用同一 Q/K/V 投影</p></div>
|
||||
<i>→</i><div class="formula-core"><code>Attn(X)</code><small>没有 mask / position</small></div><i>→</i>
|
||||
<div><span>OUTPUT</span><b>[h猫, h追, h狗]</b><p>输出跟着行顺序</p></div>
|
||||
<strong>同时重排 P</strong>
|
||||
<div><span>PERMUTED</span><b>[狗, 猫, 追]</b><p>输入行被同一个 P 重排</p></div>
|
||||
<i>→</i><div class="formula-core"><code>Attn(PX)</code><small>= P Attn(X)</small></div><i>→</i>
|
||||
<div><span>OUTPUT</span><b>[h狗, h猫, h追]</b><p>只跟随排列，不知道语序差异</p></div>
|
||||
</div>
|
||||
<div class="caution-card">
|
||||
<b>Causal mask 已经提供一部分顺序结构</b>
|
||||
<p>第 t 位只能读前缀，因此不同位置拥有不同可见集合。NoPE causal decoder 不是完全无序；但 mask 没有显式告诉 Q/K “相距多少”。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="absolute">
|
||||
<p class="section-kicker">08 / ABSOLUTE POSITION</p>
|
||||
<h2>第一代答案很直接：给每个位置一张坐标卡</h2>
|
||||
<div class="absolute-grid">
|
||||
<div class="position-card"><span>CONTENT</span><b>e<sub>token</sub></b><i style="--x:68%;--y:34%"></i></div>
|
||||
<div aria-hidden="true">＋</div>
|
||||
<div class="position-card"><span>POSITION</span><b>p<sub>37</sub></b><i style="--x:34%;--y:70%"></i></div>
|
||||
<div aria-hidden="true">＝</div>
|
||||
<div class="position-card result"><span>STATE</span><b>x<sub>37</sub></b><i style="--x:78%;--y:76%"></i></div>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<article><span>FIXED SINUSOID</span><h3>频率层级写进公式</h3><p>原始 Transformer 使用不同频率的 sin/cos，使位置间线性关系可被学习；公式可生成未见绝对位置。</p></article>
|
||||
<article><span>LEARNED TABLE</span><h3>每个位置直接学一行</h3><p>灵活但通常受训练最大索引约束；扩窗需要新增、插值或重训。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="relative">
|
||||
<p class="section-kicker">09 / RELATIVE POSITION</p>
|
||||
<h2>语言规律往往更像“离我三步”，而不是“绝对第 37 位”</h2>
|
||||
<div class="position-table">
|
||||
{positionRows.map(([name, formula, target, note]) => (
|
||||
<article><span>{name}</span><code>{formula}</code><b>{target}</b><p>{note}</p></article>
|
||||
))}
|
||||
</div>
|
||||
<p>Shaw et al. 把相对距离表示加入 self-attention；Transformer-XL 为跨段记忆重新设计相对位置；T5 将距离分桶并作为 attention bias。它们都叫“relative”，但写入的张量与参数共享方式不同。</p>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="rope">
|
||||
<p class="section-kicker">10 / ROTARY GEOMETRY</p>
|
||||
<h2>RoPE 的核心不是“旋转很高级”，而是一条矩阵恒等式</h2>
|
||||
<div class="rope-stage">
|
||||
<svg viewBox="0 0 640 360" role="img" aria-label="RoPE 将 Q K 旋转后保留相对角度">
|
||||
<circle cx="180" cy="180" r="112" /><line x1="48" y1="180" x2="312" y2="180" /><line x1="180" y1="48" x2="180" y2="312" />
|
||||
<line x1="180" y1="180" x2="276" y2="122" class="q" /><circle cx="276" cy="122" r="8" class="qdot" />
|
||||
<line x1="180" y1="180" x2="205" y2="71" class="k" /><circle cx="205" cy="71" r="8" class="kdot" />
|
||||
<path d="M235 147 A65 65 0 0 0 194 117" class="arc" />
|
||||
<text x="284" y="117">Rₘq</text><text x="211" y="65">Rₙk</text><text x="214" y="130">(n−m)θ</text>
|
||||
<text x="365" y="88">Rₘᵀ Rₙ = Rₙ₋ₘ</text>
|
||||
<text x="365" y="142">qₘᵀkₙ = qᵀRₙ₋ₘk</text>
|
||||
<text x="365" y="218">高频维度对：近距离分辨</text>
|
||||
<text x="365" y="252">低频维度对：长尺度变化</text>
|
||||
<text x="365" y="303">一个 head 同时叠加多组频率</text>
|
||||
</svg>
|
||||
<div>
|
||||
<span>2D PAIR</span>
|
||||
<code>R(mθ) = [ cos mθ −sin mθ<br /> sin mθ cos mθ ]</code>
|
||||
<p>每两维构成一个复平面。Q 在 m 旋转、K 在 n 旋转；做内积时，公共绝对角度相消，只剩相对位移 `n−m`。</p>
|
||||
<b>公式可生成任何 m ≠ 模型熟悉任何 m</b>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="extrapolation">
|
||||
<p class="section-kicker">11 / LENGTH EXTRAPOLATION</p>
|
||||
<h2>训练窗外不是“索引越界”这么简单，而是几何分布变了</h2>
|
||||
<div class="extrapolation-map">
|
||||
<div class="train-zone"><span>TRAIN / 0–8K</span><i></i><b>模型见过的相位组合与距离</b></div>
|
||||
<div class="extend-zone"><span>TEST / 8K–1M</span><i></i><b>公式可算，但组合可能陌生</b></div>
|
||||
<div class="risk-pins"><i style="left:18%">近邻</i><i style="left:42%">训练边缘</i><i style="left:73%">新周期组合</i><i style="left:92%">极长距离</i></div>
|
||||
</div>
|
||||
<div class="solution-grid">
|
||||
<article><span>CHANGE GEOMETRY</span><b>xPos · ALiBi · FIRE</b><p>重新设计距离衰减、分辨率或相对函数。</p></article>
|
||||
<article><span>MAP BACK</span><b>PI · YaRN · LongRoPE</b><p>把新位置压回熟悉范围，或按频率非均匀插值。</p></article>
|
||||
<article><span>REMOVE EXPLICIT TERM</span><b>NoPE</b><p>避免位置参数外推，但仍须证明顺序和长程能力从何而来。</p></article>
|
||||
<article><span>TRAIN THE REGIME</span><b>长数据 · 渐进扩窗</b><p>位置公式不替代远距离依赖数据与训练系统。</p></article>
|
||||
</div>
|
||||
<blockquote>
|
||||
<b>“Context window = 1M”至少有四层含义：</b>
|
||||
数学上能接收、系统上能运行、训练中见过、任务上能有效利用。任何一层都不能代替其余三层。
|
||||
</blockquote>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="nope">
|
||||
<p class="section-kicker">12 / NO POSITION ENCODING</p>
|
||||
<h2>NoPE 删除的是显式插槽，不是宇宙里的“顺序”</h2>
|
||||
<div class="nope-layers">
|
||||
<article><span>EXPLICIT PE</span><b>0</b><p>Q/K 或 embedding 不再加人工位置函数。</p></article>
|
||||
<i>但仍有</i>
|
||||
<article><span>CAUSAL PREFIX</span><b>{0…t}</b><p>每个位置的可见集合不同。</p></article>
|
||||
<article><span>RECURRENT STATE</span><b>Sₜ = f(Sₜ₋₁,xₜ)</b><p>状态更新本身带方向与时间。</p></article>
|
||||
<article><span>DATA STATISTICS</span><b>order patterns</b><p>语言序列的条件分布并不对称。</p></article>
|
||||
</div>
|
||||
<p>
|
||||
2023 的系统研究在一组 decoder-only 推理与数学任务上发现 NoPE 优于若干显式方案，并从表示能力与 SGD 模式分析它；
|
||||
这不是所有语言建模、所有架构和所有长度的普遍胜负。K3 更不能只写成“NoPE 模型”：它用 KDA 的 recurrent gating / decay 显式承担位置敏感与 recency-aware 混合。
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="deepseek-rope">
|
||||
<p class="section-kicker">13 / DEEPSEEK POSITION LINEAGE</p>
|
||||
<h2>DeepSeek 没有从 RoPE 走向“越用越多”，而是越拆越细</h2>
|
||||
<div class="deepseek-position">
|
||||
<article><span>LLM / 2024</span><h3>Full RoPE</h3><p>标准 decoder 配方：所有相关 Q/K 维度旋转。</p><b>位置与内容共处 head</b></article>
|
||||
<i>→</i>
|
||||
<article><span>V2 / MLA</span><h3>Decoupled RoPE</h3><p>content K/V 压入 latent；额外 shared K 与 multi-head Q 专门携带 RoPE。</p><b>为矩阵吸收和 KV cache 拆路</b></article>
|
||||
<i>→</i>
|
||||
<article><span>V4 / CSA·HCA</span><h3>Partial RoPE</h3><p>Q、compressed KV 与 output 的最后 64 维旋转；output 用负位置抵消绝对角。</p><b>只给部分通道位置责任</b></article>
|
||||
</div>
|
||||
<div class="mla-decoupling">
|
||||
<div><span>CONTENT PATH</span><b>h → c<sub>KV</sub> → K<sup>C</sup>, V<sup>C</sup></b><small>低秩 latent 可缓存、可吸收投影</small></div>
|
||||
<div><span>POSITION PATH</span><b>h → RoPE(K<sup>R</sup>)</b><small>额外 shared key + per-head query</small></div>
|
||||
<div><span>ATTENTION KEY</span><b>[K<sup>C</sup>; K<sup>R</sup>]</b><small>内容与位置在 score 前拼接</small></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="norm-object">
|
||||
<p class="section-kicker">14 / NORMALIZATION TARGET</p>
|
||||
<h2>看到“RMSNorm”先别点头：第一问永远是“它在归一化谁？”</h2>
|
||||
<div class="norm-table">
|
||||
<div class="norm-row head"><span>方法 / 插槽</span><span>对象</span><span>动作</span><span>主要目的</span></div>
|
||||
{normRows.map(([name, target, action, purpose]) => (
|
||||
<div class="norm-row"><b>{name}</b><span>{target}</span><code>{action}</code><span>{purpose}</span></div>
|
||||
))}
|
||||
</div>
|
||||
<div class="formula-duo">
|
||||
<article><span>LAYERNORM</span><code>LN(x) = γ ⊙ (x−μ) / √(σ²+ε) + β</code><p>同时获得平移和缩放不变性；需要均值与方差。</p></article>
|
||||
<article><span>RMSNORM</span><code>RMS(x) = γ ⊙ x / √(mean(x²)+ε)</code><p>保留均值方向，只控制 root-mean-square 尺度。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="norm-topology">
|
||||
<p class="section-kicker">15 / PRE VS POST</p>
|
||||
<h2>Norm 的位置会改变那条“什么都不做也能通过”的路</h2>
|
||||
<div class="topology-compare">
|
||||
<article>
|
||||
<span>POST-LN / 2017 TRANSFORMER</span>
|
||||
<code>xₗ₊₁ = Norm(xₗ + F(xₗ))</code>
|
||||
<div class="block-glyph post"><b>x</b><i>F</i><em>＋</em><strong>Norm</strong><b>y</b></div>
|
||||
<p>相加后尺度被重置，但 identity path 每层都穿过 Norm；初始化靠近输出处的梯度可能更大。</p>
|
||||
</article>
|
||||
<article>
|
||||
<span>PRE-LN / MODERN DECODER</span>
|
||||
<code>xₗ₊₁ = xₗ + F(Norm(xₗ))</code>
|
||||
<div class="block-glyph pre"><b>x</b><strong>Norm</strong><i>F</i><em>＋</em><b>y</b></div>
|
||||
<p>裸 identity path 直接跨层，通常更易优化；所有分支固定单位累加，幅值和层贡献稀释成为新问题。</p>
|
||||
</article>
|
||||
</div>
|
||||
<blockquote>
|
||||
<b>Pre-LN “更稳”不等于“全面更优”。</b>
|
||||
初始化梯度、是否需要 warm-up、最终表示变化、深度收益与下游质量是不同指标。NormFormer、DeepNorm、AttnRes 正是沿不同缺口继续修补。
|
||||
</blockquote>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="deep-stack">
|
||||
<p class="section-kicker">16 / TRAINING DEEP</p>
|
||||
<h2>从“让梯度能走”到“让每层的贡献可控”</h2>
|
||||
<div class="deep-stack-timeline">
|
||||
{deepStack.map(([year, title, mechanism, note]) => (
|
||||
<article><span>{year}</span><h3>{title}</h3><b>{mechanism}</b><p>{note}</p></article>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="residual">
|
||||
<p class="section-kicker">17 / RESIDUAL STREAM</p>
|
||||
<h2>Residual 解决“到不了”，却没有自动解决“听不见”</h2>
|
||||
<div class="residual-equation">
|
||||
<span>PRENORM UNROLLED</span>
|
||||
<code>h<sub>L</sub> = h<sub>0</sub> + f<sub>0</sub>(Norm(h<sub>0</sub>)) + f<sub>1</sub>(Norm(h<sub>1</sub>)) + … + f<sub>L−1</sub>(Norm(h<sub>L−1</sub>))</code>
|
||||
<div class="residual-bars">
|
||||
{Array.from({length:12},(_,i) => <i style={`--h:${28 + i*5}%`}><small>L{i}</small></i>)}
|
||||
</div>
|
||||
<p>上图是“累计 state 变大时，固定大小单层增量相对占比下降”的概念图，不是实测激活。Kimi AttnRes 报告把这类现象称为 hidden-state growth 与 PreNorm dilution。</p>
|
||||
</div>
|
||||
<div class="three-questions">
|
||||
<article><span>REACHABILITY</span><b>信息能否跨层？</b><p>identity shortcut 的原始强项。</p></article>
|
||||
<article><span>MAGNITUDE</span><b>累计 state 会多大？</b><p>Norm、初始化、scale、gate 共同控制。</p></article>
|
||||
<article><span>SELECTIVITY</span><b>当前层想读哪一层？</b><p>标准加法没有选择；HC / AttnRes 才重写拓扑。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="hc">
|
||||
<p class="section-kicker">18 / HYPER-CONNECTIONS</p>
|
||||
<h2>把一条 residual stream 展开成四条“车道”</h2>
|
||||
<div class="hc-diagram">
|
||||
<div class="stream-bank">
|
||||
<span>Xₗ ∈ R<sup>n×d</sup></span>
|
||||
<i>stream 1</i><i>stream 2</i><i>stream 3</i><i>stream 4</i>
|
||||
</div>
|
||||
<div class="mapping"><b>Aₗ</b><small>4 streams → layer input d</small></div>
|
||||
<div class="layer-f"><b>Fₗ</b><small>Attention / MoE</small></div>
|
||||
<div class="mapping"><b>Cₗ</b><small>layer output → 4 streams</small></div>
|
||||
<div class="mix-matrix"><b>Bₗ</b><small>stream-to-stream mixing</small></div>
|
||||
</div>
|
||||
<div class="formula-panel">
|
||||
<span>HYPER-CONNECTION UPDATE</span>
|
||||
<code>X<sub>l+1</sub> = B<sub>l</sub>X<sub>l</sub> + C<sub>l</sub>F<sub>l</sub>(A<sub>l</sub>X<sub>l</sub>)</code>
|
||||
<p>实际 layer 仍只接收 d 维输入；新增的是 residual width 与连接自由度。原论文将目标描述为缓解 gradient vanishing 与 representation collapse 的跷跷板。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="mhc">
|
||||
<p class="section-kicker">19 / MANIFOLD CONSTRAINT</p>
|
||||
<h2>DeepSeek 的关键追问：自由连接变多后，identity property 怎么回来？</h2>
|
||||
<div class="mhc-stage">
|
||||
<div class="raw-matrix">
|
||||
<span>RAW B̃</span>
|
||||
<div>{[.8,-.4,.2,.9,.6,.1,-.7,.3,.5,.2,.4,-.1,.9,.2,.1,.6].map(v=><i style={`--v:${Math.abs(v)}`}>{v}</i>)}</div>
|
||||
<b>可能放大 / 抵消</b>
|
||||
</div>
|
||||
<div class="sinkhorn"><span>PROJECT</span><b>exp → row norm → col norm</b><i>× 20</i><small>DeepSeek-V4 实际设置</small></div>
|
||||
<div class="stable-matrix">
|
||||
<span>B ∈ BIRKHOFF POLYTOPE</span>
|
||||
<div>{[.42,.18,.25,.15,.16,.48,.12,.24,.22,.14,.46,.18,.20,.20,.17,.43].map(v=><i style={`--v:${v}`}>{v}</i>)}</div>
|
||||
<b>非负 · 行和=1 · 列和=1</b>
|
||||
</div>
|
||||
</div>
|
||||
<div class="proof-cards">
|
||||
<article><span>NON-EXPANSIVE</span><b>‖B‖₂ ≤ 1</b><p>报告据此限制 forward / backward 中 residual transformation 的扩张。</p></article>
|
||||
<article><span>CLOSED UNDER PRODUCT</span><b>B₁B₂ ∈ 𝓜</b><p>doubly stochastic 集合在乘法下封闭，支持深层连续堆叠。</p></article>
|
||||
<article><span>BOUNDED A / C</span><b>Sigmoid constraints</b><p>输入和输出映射非负且有界，降低信号抵消风险。</p></article>
|
||||
</div>
|
||||
<p class="source-line">以上是 <a href="https://arxiv.org/abs/2512.24880">mHC 论文</a> 与 <a href="https://arxiv.org/abs/2606.19348">DeepSeek-V4 报告</a> 的设计和理论主张；本站不把它扩写成任意优化器与任意网络的完备稳定性证明。</p>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="attnres">
|
||||
<p class="section-kicker">20 / ATTENTION OVER DEPTH</p>
|
||||
<h2>Kimi 的答案不同：不扩宽车道，而是让当前层选择历史出口</h2>
|
||||
<div class="attnres-diagram">
|
||||
<div class="history-layers">
|
||||
<div><span>EMB</span><b>v₀</b></div><div><span>L1</span><b>v₁</b></div><div><span>L2</span><b>v₂</b></div><div><span>…</span><b>…</b></div><div><span>L−1</span><b>vₗ₋₁</b></div>
|
||||
</div>
|
||||
<div class="depth-softmax">
|
||||
<span>LAYER-SPECIFIC QUERY qₗ</span>
|
||||
<div><i style="--w:24%">.08</i><i style="--w:42%">.15</i><i style="--w:90%">.36</i><i style="--w:30%">.10</i><i style="--w:76%">.31</i></div>
|
||||
<code>αᵢ→ₗ = softmax(qₗᵀ RMSNorm(kᵢ))</code>
|
||||
</div>
|
||||
<div class="selected-state"><span>CURRENT INPUT</span><b>hₗ = Σ αᵢ→ₗvᵢ</b><p>内容相关、归一化的深度混合</p></div>
|
||||
</div>
|
||||
<div class="full-block-compare">
|
||||
<article><span>FULL ATTNRES</span><b>保存全部层输出</b><code>memory O(Ld)</code><p>depth arithmetic 为 O(L²d)；不足百层时算术可接受，训练状态与 PP 通信更难。</p></article>
|
||||
<article><span>BLOCK ATTNRES</span><b>块内求和、块间 Attention</b><code>memory O(Nd)</code><p>只保存 block representations，保留大部分选择性并降低跨 stage 状态。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="k3-block">
|
||||
<p class="section-kicker">21 / K3 DEPTH MAP</p>
|
||||
<h2>93 层不是 93 份同时常驻：K3 把它们压成 9 个深度来源</h2>
|
||||
<div class="k3-depth-map">
|
||||
<div class="embedding-source"><span>SOURCE 0</span><b>Token Embedding</b><small>始终可被读取</small></div>
|
||||
{Array.from({length:8},(_,i) => (
|
||||
<article class={i===7?"partial":""}>
|
||||
<span>SOURCE {i+1}</span>
|
||||
<b>{i===7?"L85–L93":"L"+(i*12+1)+"–L"+((i+1)*12)}</b>
|
||||
<div>{Array.from({length:i===7?9:12},(_,j)=><i title={`Layer ${i*12+j+1}`}></i>)}</div>
|
||||
<small>{i===7?"partial / 9 layers":"12-layer block"}</small>
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
<div class="correction-note">
|
||||
<b>勘误 / 本章研究阶段已更正</b>
|
||||
<p>K3 §2.2 明确写的是 12-layer block size：93 层形成 7 个完整块和 1 个尾块，另计 embedding 后共有 9 个 block-level sources。“block size 2”是早期草案误读，未进入正式页面结论。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="activation">
|
||||
<p class="section-kicker">22 / ACTIVATION LINEAGE</p>
|
||||
<h2>激活函数的演化，不只是把 ReLU 曲线画得更圆</h2>
|
||||
<div class="activation-table">
|
||||
<div class="activation-row head"><span>方法</span><span>公式</span><span>结构</span><span>极值</span></div>
|
||||
{activationRows.map(([name, formula, structure, bound]) => (
|
||||
<div class="activation-row"><b>{name}</b><code>{formula}</code><span>{structure}</span><span>{bound}</span></div>
|
||||
))}
|
||||
</div>
|
||||
<div class="glu-anatomy">
|
||||
<div><span>GATE BRANCH</span><b>g = W<sub>g</sub>x</b><i>Swish(g)</i></div>
|
||||
<strong>⊙</strong>
|
||||
<div><span>VALUE BRANCH</span><b>u = W<sub>u</sub>x</b><i>linear u</i></div>
|
||||
<strong>→</strong>
|
||||
<div class="danger"><span>SwiGLU</span><b>unbounded × unbounded</b><i>大规模 / MoE / 低精度下会放大 outlier 风险</i></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="outliers">
|
||||
<p class="section-kicker">23 / OUTLIER CONTROL</p>
|
||||
<h2>DeepSeek-V4 与 K3 都限制 SwiGLU 极值，但数学性格完全不同</h2>
|
||||
<div class="outlier-compare">
|
||||
<article class="deepseek-card">
|
||||
<span>DEEPSEEK-V4 / HARD BOUNDARY</span>
|
||||
<h3>SwiGLU Clamping</h3>
|
||||
<code>gate = min(Swish(g), 10)<br />value = clamp(u, −10, 10)</code>
|
||||
<div class="clip-curve"><i></i></div>
|
||||
<p>报告称该配方消除 outliers、帮助稳定且不损性能。这是 V4 训练中的作者经验结论；硬 clamp 在边界外对被截分支给出零梯度。</p>
|
||||
</article>
|
||||
<article class="kimi-card">
|
||||
<span>KIMI K3 / SMOOTH BOUNDARY</span>
|
||||
<h3>Sigmoid Tanh Unit GLU</h3>
|
||||
<code>4 tanh(g/4) · σ(g)<br />× 25 tanh(u/25)</code>
|
||||
<div class="smooth-curve"><i></i></div>
|
||||
<p>两支都用 tanh 平滑压缩；β₁=4、β₂=25，所以逐点 |y|<100。靠近原点一阶近似 SwiGLU，β→∞ 时逐点恢复。</p>
|
||||
</article>
|
||||
</div>
|
||||
<div class="gradient-note">
|
||||
<span>FUNCTION PROPERTY ≠ WHOLE-NETWORK GUARANTEE</span>
|
||||
<p>SiTU 单元输出有界，不等于整个 residual stream、loss 或 gradient 有界；K3 还在 routed expert aggregate 后加入 RMSNorm，并配合路由与系统稳定措施。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="deepseek">
|
||||
<p class="section-kicker">24 / DEEPSEEK FOCUS</p>
|
||||
<h2>沿 DeepSeek 六个节点，看四条轴如何逐步解耦再合流</h2>
|
||||
<div class="deepseek-ledger">
|
||||
{deepseekRows.map(([model, recipe, lesson]) => (
|
||||
<article><span>{model}</span><h3>{recipe}</h3><p>{lesson}</p></article>
|
||||
))}
|
||||
</div>
|
||||
<div class="v4-control-room">
|
||||
<div class="v4-center"><span>DEEPSEEK-V4 BLOCK</span><b>表示稳定控制室</b></div>
|
||||
<article style="--x:6%;--y:8%"><span>DEPTH</span><b>mHC ×4</b><p>约束流间映射</p></article>
|
||||
<article style="--x:68%;--y:8%"><span>MATCH</span><b>Q / KV RMSNorm</b><p>控制 logits</p></article>
|
||||
<article style="--x:6%;--y:66%"><span>POSITION</span><b>last 64 dims RoPE</b><p>partial rotation</p></article>
|
||||
<article style="--x:68%;--y:66%"><span>FFN</span><b>SwiGLU clamp</b><p>控制 outliers</p></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="k3">
|
||||
<p class="section-kicker">25 / KIMI K3 CONVERGENCE</p>
|
||||
<h2>K3 的四轴不是四个 patch，而是一条完整信息路径</h2>
|
||||
<div class="k3-flow">
|
||||
<article><span>01 / TOKEN</span><b>160K vocabulary</b><p>文本与视觉 token 进入共享 backbone；接口参数和多模态配方共同决定入口。</p></article>
|
||||
<i>→</i>
|
||||
<article><span>02 / SEQUENCE</span><b>3× KDA + 1× Gated MLA</b><p>KDA recurrent decay 提供顺序与近因；MLA NoPE 提供周期性全局内容交互。</p></article>
|
||||
<i>→</i>
|
||||
<article><span>03 / DEPTH</span><b>Block AttnRes</b><p>93 层压成 8 个 layer blocks + embedding，共 9 个可选深度来源。</p></article>
|
||||
<i>→</i>
|
||||
<article><span>04 / LOCAL</span><b>Stable LatentMoE</b><p>routed latent RMSNorm + SiTU-GLU，把稀疏专家路径的尺度和乘法极值一起控制。</p></article>
|
||||
</div>
|
||||
<div class="k3-nope-evidence">
|
||||
<span>K3 §3.4 / LONG-CONTEXT BOUNDARY</span>
|
||||
<h3>NoPE 避免“修改位置参数”，却没有避免训练长上下文</h3>
|
||||
<div>
|
||||
<b>8K</b><i></i><b>64K</b><i></i><b>256K</b><i></i><b>1M</b>
|
||||
</div>
|
||||
<p>报告同时使用长文档 / 视频清洗与合成、渐进扩窗课程和序列维并行。因而“K3 直接外推到 1M”不能被简化成“NoPE 单因收益”。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section lab-section" id="lab">
|
||||
<p class="section-kicker">26 / INTERACTIVE LAB</p>
|
||||
<h2>现在亲手改四个变量：单位、位置、深度、极值</h2>
|
||||
<p class="lead">每台仪器都把论文公式和本站 toy assumption 分开标注。先找方向关系，再回到真实报告核对训练与系统边界。</p>
|
||||
<RepresentationLab />
|
||||
</section>
|
||||
|
||||
<section class="article-section" id="audit">
|
||||
<p class="section-kicker">27 / ARCHITECTURE AUDIT</p>
|
||||
<h2>以后再看任何架构图，用这十二问防止“名字读懂、路径读错”</h2>
|
||||
<div class="audit-grid">
|
||||
{auditRows.map(([title, question], index) => (
|
||||
<article><span>{String(index + 1).padStart(2,"0")}</span><h3>{title}</h3><p>{question}</p></article>
|
||||
))}
|
||||
</div>
|
||||
<div class="final-thesis">
|
||||
<span>THE CHAPTER IN ONE MAP</span>
|
||||
<h3>Token 决定“一步”，Position 决定“先后”，Norm 决定“尺度”，Residual 决定“深度可达”，FFN 决定“逐位置改写”。</h3>
|
||||
<p>现代 LLM 的架构创新，往往不是发明一个孤立 block，而是重新分配这五种责任，再让算法、低精度、缓存和通信能够承受它。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="article-section paper-section" id="papers">
|
||||
<p class="section-kicker">PAPER CHAIN / 2003–2026</p>
|
||||
<h2>六十六个节点，不按热度，按“前一篇留下什么问题”排列</h2>
|
||||
<p class="lead">页面只写每个节点在本章问题链上的位置；点击进入论文原文或正式页面。具体数字不脱离模型、任务和训练设定外推。</p>
|
||||
<div class="paper-chain">
|
||||
{paperChain.map(([year, title, href, note], index) => (
|
||||
<a href={href}>
|
||||
<span>{String(index + 1).padStart(2,"0")} / {year}</span>
|
||||
<h3>{title}</h3>
|
||||
<p>{note}</p>
|
||||
<b aria-hidden="true">↗</b>
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
</article>
|
||||
</div>
|
||||
</BaseLayout>
|
||||
|
||||
<style>
|
||||
.chapter-hero { display:grid; grid-template-columns:1fr 340px; min-height:650px; color:#f4f0e8; background:linear-gradient(135deg,#172437 0%,#263a50 72%,#3c4a57 100%); }
|
||||
.chapter-hero-main { display:flex; flex-direction:column; justify-content:center; padding:80px max(5vw,46px); }
|
||||
.chapter-hero h1 { max-width:900px; margin:24px 0; color:#fff; font:650 clamp(3.1rem,8vw,7.2rem)/.88 var(--serif); letter-spacing:-.055em; }
|
||||
.chapter-hero h1 em { color:#d6a072; font-style:normal; }
|
||||
.chapter-deck { max-width:840px; color:#c0cad4; font-size:.8rem; line-height:1.9; }
|
||||
.hero-thesis { max-width:850px; margin-top:42px; padding:20px 0 0; border-top:1px solid #53667b; }
|
||||
.hero-thesis span,.chapter-hero-aside > span { color:#d6a072; font:600 .55rem var(--mono); letter-spacing:.14em; }
|
||||
.hero-thesis p { margin:10px 0 0; color:#d8dfe5; font-size:.68rem; line-height:1.7; }.hero-thesis b { color:#fff; }
|
||||
.chapter-hero-aside { display:flex; flex-direction:column; justify-content:center; padding:44px 30px; border-left:1px solid #53667b; background:rgba(10,21,34,.34); }
|
||||
.chapter-hero-aside strong { display:block; margin:22px 0 5px; color:#fff; font:650 2.3rem var(--serif); }
|
||||
.chapter-hero-aside > p { color:#aebac6; font-size:.58rem; line-height:1.7; }
|
||||
.chapter-hero-aside dl { margin:38px 0 0; }.chapter-hero-aside dl div { display:flex; justify-content:space-between; gap:14px; padding:12px 0; border-top:1px solid #44576b; }
|
||||
.chapter-hero-aside dt { color:#8fa0b1; font:.47rem var(--mono); }.chapter-hero-aside dd { margin:0; font:600 .58rem var(--sans); }
|
||||
.article-shell { display:grid; grid-template-columns:230px minmax(0,1fr); max-width:1600px; margin:auto; }
|
||||
.side-rail { position:sticky; top:0; align-self:start; height:100vh; overflow:auto; padding:35px 22px; border-right:1px solid var(--line); background:var(--paper-raised); }
|
||||
.side-rail > span { color:var(--copper); font:600 .5rem var(--mono); letter-spacing:.14em; }
|
||||
.side-rail nav { display:flex; flex-direction:column; margin-top:17px; }
|
||||
.side-rail a { display:grid; grid-template-columns:28px 1fr; gap:7px; padding:7px 0; color:var(--muted); font-size:.49rem; line-height:1.35; text-decoration:none; }
|
||||
.side-rail a:hover { color:var(--ink); }.side-rail a b { color:var(--copper); font:.45rem var(--mono); }
|
||||
.article { min-width:0; padding:0 clamp(28px,5vw,82px) 100px; }
|
||||
.article-section { max-width:1180px; margin:0 auto; padding:82px 0; border-bottom:1px solid var(--line); scroll-margin-top:20px; }
|
||||
.section-kicker { margin:0 0 17px; color:var(--copper); font:600 .56rem var(--mono); letter-spacing:.14em; }
|
||||
.article-section h2 { max-width:980px; margin:0 0 25px; font:650 clamp(1.8rem,4vw,3.25rem)/1.08 var(--serif); letter-spacing:-.03em; }
|
||||
.article-section > p:not(.section-kicker,.lead,.source-line) { max-width:920px; color:var(--muted); font-size:.7rem; line-height:1.9; }
|
||||
.lead { max-width:980px; margin:0 0 30px; color:var(--ink-soft); font-size:.78rem; line-height:1.9; }
|
||||
.article-section blockquote { margin:28px 0 0; padding:22px 25px; border-left:3px solid var(--copper); background:rgba(193,124,68,.08); color:var(--muted); font-size:.67rem; line-height:1.85; }
|
||||
.article-section blockquote b { color:var(--ink); }
|
||||
.rep-ledgers { display:grid; grid-template-columns:repeat(4,1fr); }
|
||||
.rep-ledgers article { min-height:205px; padding:20px; border:1px solid var(--line); border-right:0; border-bottom:0; background:var(--paper-raised); }
|
||||
.rep-ledgers article:nth-child(4n) { border-right:1px solid var(--line); }.rep-ledgers article:nth-last-child(-n+4) { border-bottom:1px solid var(--line); }
|
||||
.rep-ledgers span,.four-axis span,.object-stack article > span,.tradeoff-triangle span,.evidence-note > span,.context-story span,.attention-ffn span,.symmetry-proof span,.two-col span,.solution-grid span,.caution-card span,.deepseek-position span,.mla-decoupling span,.formula-duo span,.topology-compare span,.three-questions span,.hc-diagram span,.proof-cards span,.history-layers span,.depth-softmax span,.selected-state span,.full-block-compare span,.k3-depth-map span,.correction-note span,.glu-anatomy span,.outlier-compare span,.gradient-note span,.deepseek-ledger span,.v4-control-room span,.k3-flow span,.k3-nope-evidence span,.final-thesis span { color:var(--copper); font:600 .49rem var(--mono); letter-spacing:.1em; }
|
||||
.rep-ledgers h3 { margin:14px 0 9px; font:650 .9rem var(--serif); }.rep-ledgers p { min-height:52px; color:var(--muted); font-size:.56rem; line-height:1.55; }.rep-ledgers b { font:600 .54rem/1.5 var(--sans); }
|
||||
.four-axis { display:grid; grid-template-columns:repeat(4,1fr); gap:10px; margin-top:22px; }.four-axis div { min-height:155px; padding:19px; color:#fff; background:#172437; }.four-axis b { display:block; margin:25px 0 9px; font:650 .82rem var(--serif); }.four-axis p { color:#aebac7; font-size:.52rem; line-height:1.55; }
|
||||
.object-stack { border:1px solid var(--line-strong); }.object-stack article { display:grid; grid-template-columns:55px 270px 1fr; gap:18px; align-items:center; min-height:96px; padding:16px 20px; border-bottom:1px solid var(--line); }.object-stack article:last-child { border-bottom:0; }.object-stack article > span { font-size:.65rem; }.object-stack b { display:block; font:650 .8rem var(--serif); }.object-stack code { color:var(--copper); font:.55rem var(--mono); }.object-stack p { margin:0; color:var(--muted); font-size:.62rem; line-height:1.6; }
|
||||
.formula-panel { margin:25px 0 0; padding:25px; color:#fff; background:#172437; }.formula-panel > span { color:#d49a68; font:.5rem var(--mono); letter-spacing:.1em; }.formula-panel code { display:block; margin:22px 0 13px; color:#eef2f5; font:.72rem var(--mono); white-space:normal; }.formula-panel p { margin:0; color:#aebac7; font-size:.58rem; line-height:1.65; }
|
||||
.tradeoff-triangle { display:grid; grid-template-columns:repeat(3,1fr); gap:1px; background:var(--line-strong); }.tradeoff-triangle article { min-height:205px; padding:25px; background:var(--paper-raised); }.tradeoff-triangle b { display:block; margin:30px 0 12px; font:650 1.4rem var(--serif); }.tradeoff-triangle p { color:var(--muted); font-size:.6rem; line-height:1.65; }
|
||||
.unit-ladder { display:grid; grid-template-columns:repeat(4,1fr); margin:22px 0; }.unit-ladder div { position:relative; min-height:125px; padding:18px; border:1px solid var(--line); border-right:0; }.unit-ladder div:last-child { border-right:1px solid var(--line); }.unit-ladder span { color:var(--copper); font:.5rem var(--mono); }.unit-ladder b { display:block; margin-top:30px; font:650 .7rem var(--serif); }.unit-ladder i { position:absolute; right:-12px; top:50%; z-index:2; width:23px; height:23px; border:1px solid var(--line); border-left:0; border-bottom:0; background:var(--paper); transform:translateY(-50%) rotate(45deg); }
|
||||
.evidence-note { display:grid; grid-template-columns:150px 1fr auto; gap:22px; align-items:center; padding:20px; border:1px solid var(--line); background:rgba(193,124,68,.06); }.evidence-note p { margin:0; color:var(--muted); font-size:.6rem; line-height:1.65; }.evidence-note a,.source-line a { color:var(--copper); font-size:.55rem; }
|
||||
.tying-diagram { position:relative; display:grid; grid-template-columns:1fr 35px 1.2fr 35px 1fr; gap:6px; align-items:center; min-height:250px; padding:32px 32px 85px; border:1px solid var(--line-strong); }.tying-diagram > div { min-height:110px; padding:19px; border:1px solid var(--line); background:var(--paper-raised); }.tying-diagram > i { color:var(--copper); text-align:center; font-style:normal; }.tying-diagram span { color:var(--copper); font:.48rem var(--mono); }.tying-diagram b { display:block; margin:17px 0 7px; font:650 1rem var(--serif); }.tying-diagram small { color:var(--muted); font-size:.52rem; }.tying-diagram svg { position:absolute; inset:auto 5% 4px; width:90%; height:100px; }.tying-diagram svg path { fill:none; stroke:var(--copper); stroke-width:1.5; }.tying-diagram svg text { fill:var(--copper); font:10px var(--mono); }
|
||||
.comparison-table { margin-top:18px; border:1px solid var(--line); }.table-row { display:grid; grid-template-columns:140px 120px 1fr 1fr; border-bottom:1px solid var(--line); }.table-row:last-child { border-bottom:0; }.table-row > * { padding:14px; border-right:1px solid var(--line); font-size:.58rem; }.table-row > *:last-child { border-right:0; }.table-row.head { color:var(--muted); background:var(--paper-raised); font:.48rem var(--mono); }
|
||||
.source-line { margin-top:20px; color:var(--muted); font-size:.58rem; }
|
||||
.context-story { display:grid; grid-template-columns:1fr 28px 1fr 28px 1fr 28px 1fr; align-items:stretch; }.context-story article { min-height:205px; padding:20px; border:1px solid var(--line); background:var(--paper-raised); }.context-story > i { display:grid; place-items:center; color:var(--copper); font-style:normal; }.context-story h3 { margin:22px 0 10px; font:650 .88rem var(--serif); }.context-story p { min-height:65px; color:var(--muted); font-size:.56rem; line-height:1.6; }.context-story article > b { font:.5rem var(--mono); }
|
||||
.attention-ffn { display:grid; grid-template-columns:1fr 50px 1fr; align-items:center; }.attention-ffn article { min-height:270px; padding:26px; border:1px solid var(--line-strong); background:var(--paper-raised); }.attention-ffn > div { text-align:center; color:var(--copper); font:1.5rem var(--serif); }.attention-ffn h3 { margin:20px 0; font:650 1.2rem var(--serif); }.attention-ffn p { color:var(--muted); font-size:.6rem; line-height:1.65; }.token-row { display:flex; gap:8px; }.token-row i,.ffn-glyph i { display:grid; place-items:center; width:55px; height:55px; border:1px solid var(--line); background:var(--paper); font:650 .65rem var(--serif); }.ffn-glyph { display:flex; align-items:center; gap:10px; }.ffn-glyph b { color:var(--copper); }
|
||||
.wave-timeline { position:relative; display:grid; grid-template-columns:repeat(2,1fr); gap:12px; }.wave-timeline article { min-height:210px; padding:24px; border:1px solid var(--line); background:var(--paper-raised); }.wave-timeline span { color:var(--copper); font:.5rem var(--mono); }.wave-timeline h3 { margin:20px 0 9px; font:650 1rem var(--serif); }.wave-timeline b { font:.52rem var(--mono); }.wave-timeline p { color:var(--muted); font-size:.58rem; line-height:1.65; }
|
||||
.symmetry-proof { display:grid; grid-template-columns:1fr 30px .8fr 30px 1fr; gap:5px; align-items:center; padding:26px; border:1px solid var(--line-strong); }.symmetry-proof > div { padding:18px; border:1px solid var(--line); background:var(--paper-raised); }.symmetry-proof > i { color:var(--copper); text-align:center; font-style:normal; }.symmetry-proof > strong { grid-column:1/-1; padding:10px; color:#fff; background:var(--navy); text-align:center; font:.52rem var(--mono); }.symmetry-proof b,.symmetry-proof code { display:block; margin:15px 0 7px; font:650 .75rem var(--serif); }.symmetry-proof p,.symmetry-proof small { margin:0; color:var(--muted); font-size:.52rem; }.formula-core { text-align:center; }
|
||||
.caution-card { margin-top:18px; padding:20px; border-left:3px solid var(--copper); background:rgba(193,124,68,.08); }.caution-card b { font:650 .78rem var(--serif); }.caution-card p { margin:9px 0 0; color:var(--muted); font-size:.6rem; line-height:1.65; }
|
||||
.absolute-grid { display:grid; grid-template-columns:1fr 40px 1fr 40px 1fr; align-items:center; }.absolute-grid > div[aria-hidden] { color:var(--copper); text-align:center; font:1.4rem var(--serif); }.position-card { position:relative; min-height:220px; padding:20px; border:1px solid var(--line); background:radial-gradient(circle at var(--x) var(--y),rgba(193,124,68,.27) 0 8%,transparent 9%),var(--paper-raised); }.position-card span { color:var(--copper); font:.5rem var(--mono); }.position-card b { position:absolute; inset:50% auto auto 50%; transform:translate(-50%,-50%); font:650 1.4rem var(--serif); }.position-card i { position:absolute; left:var(--x); top:var(--y); width:9px; height:9px; border-radius:50%; background:var(--copper); }.position-card.result { color:#fff; background:radial-gradient(circle at var(--x) var(--y),rgba(212,160,114,.5) 0 8%,transparent 9%),#172437; }
|
||||
.two-col { display:grid; grid-template-columns:1fr 1fr; gap:12px; margin-top:20px; }.two-col article { padding:22px; border:1px solid var(--line); }.two-col h3 { margin:20px 0 10px; font:650 .9rem var(--serif); }.two-col p { color:var(--muted); font-size:.58rem; line-height:1.65; }
|
||||
.position-table { display:grid; grid-template-columns:repeat(5,1fr); }.position-table article { min-height:220px; padding:19px; border:1px solid var(--line); border-right:0; background:var(--paper-raised); }.position-table article:last-child { border-right:1px solid var(--line); }.position-table span { color:var(--copper); font:.5rem var(--mono); }.position-table code { display:block; min-height:42px; margin:22px 0 12px; font:.58rem var(--mono); white-space:normal; }.position-table b { font:.52rem var(--mono); }.position-table p { color:var(--muted); font-size:.53rem; line-height:1.55; }
|
||||
.rope-stage { display:grid; grid-template-columns:1.2fr .8fr; color:#fff; background:#172437; }.rope-stage svg { width:100%; min-height:360px; }.rope-stage svg circle:first-child { fill:none; stroke:#506277; }.rope-stage svg line { stroke:#43566b; }.rope-stage svg .q { stroke:#d69a68; stroke-width:3; }.rope-stage svg .k { stroke:#eef2f5; stroke-width:3; }.rope-stage svg .qdot { fill:#d69a68; }.rope-stage svg .kdot { fill:#eef2f5; }.rope-stage svg .arc { fill:none; stroke:#95a7b8; stroke-dasharray:4 3; }.rope-stage svg text { fill:#c2cbd4; font:13px var(--mono); }.rope-stage > div { padding:35px; border-left:1px solid #485b70; }.rope-stage > div span { color:#d49a68; font:.5rem var(--mono); }.rope-stage code { display:block; margin:30px 0; font:.67rem/1.8 var(--mono); }.rope-stage p { color:#aebac7; font-size:.6rem; line-height:1.7; }.rope-stage b { display:block; margin-top:28px; color:#d49a68; font:650 .75rem var(--serif); }
|
||||
.extrapolation-map { position:relative; display:grid; grid-template-columns:1fr 3fr; min-height:185px; margin-bottom:22px; border:1px solid var(--line-strong); }.extrapolation-map > div:not(.risk-pins) { padding:20px; }.train-zone { background:rgba(193,124,68,.12); }.extend-zone { background:#172437; color:#fff; }.extrapolation-map span { color:var(--copper); font:.5rem var(--mono); }.extrapolation-map b { display:block; margin-top:70px; font:650 .72rem var(--serif); }.risk-pins { position:absolute; inset:auto 0 12px; }.risk-pins i { position:absolute; bottom:0; color:var(--muted); font:8px var(--mono); font-style:normal; }.risk-pins i::before { content:""; display:block; width:1px; height:45px; margin:auto; background:var(--copper); }
|
||||
.solution-grid { display:grid; grid-template-columns:repeat(4,1fr); gap:9px; }.solution-grid article { min-height:160px; padding:18px; border:1px solid var(--line); background:var(--paper-raised); }.solution-grid b { display:block; margin:22px 0 9px; font:650 .72rem var(--serif); }.solution-grid p { color:var(--muted); font-size:.53rem; line-height:1.55; }
|
||||
.nope-layers { display:grid; grid-template-columns:1fr 70px repeat(3,1fr); align-items:stretch; }.nope-layers article { padding:20px; border:1px solid var(--line); background:var(--paper-raised); }.nope-layers > i { display:grid; place-items:center; color:var(--copper); font:.55rem var(--serif); }.nope-layers b { display:block; margin:22px 0 9px; font:650 .8rem var(--serif); }.nope-layers p { color:var(--muted); font-size:.52rem; line-height:1.55; }
|
||||
.deepseek-position { display:grid; grid-template-columns:1fr 35px 1fr 35px 1fr; align-items:stretch; }.deepseek-position article { min-height:240px; padding:23px; border:1px solid var(--line); background:var(--paper-raised); }.deepseek-position > i { display:grid; place-items:center; color:var(--copper); font-style:normal; }.deepseek-position h3 { margin:25px 0 12px; font:650 1.05rem var(--serif); }.deepseek-position p { min-height:74px; color:var(--muted); font-size:.56rem; line-height:1.6; }.deepseek-position article > b { font:.5rem var(--mono); }
|
||||
.mla-decoupling { display:grid; grid-template-columns:1fr 1fr 1fr; margin-top:18px; }.mla-decoupling div { min-height:135px; padding:18px; color:#fff; background:#172437; border-right:1px solid #4a5d72; }.mla-decoupling b { display:block; margin:24px 0 7px; font:650 .74rem var(--serif); }.mla-decoupling small { color:#aebac7; font-size:.5rem; }
|
||||
.norm-table,.activation-table { border:1px solid var(--line); }.norm-row,.activation-row { display:grid; grid-template-columns:170px 1.2fr 1fr 1fr; }.norm-row > *,.activation-row > * { padding:15px; border-right:1px solid var(--line); border-bottom:1px solid var(--line); font-size:.56rem; }.norm-row > *:last-child,.activation-row > *:last-child { border-right:0; }.norm-row:last-child > *,.activation-row:last-child > * { border-bottom:0; }.norm-row.head,.activation-row.head { color:var(--muted); background:var(--paper-raised); font:.48rem var(--mono); }.norm-row code,.activation-row code { font:.52rem var(--mono); white-space:normal; }
|
||||
.formula-duo { display:grid; grid-template-columns:1fr 1fr; gap:10px; margin-top:18px; }.formula-duo article { padding:22px; border:1px solid var(--line); }.formula-duo code { display:block; margin:22px 0 12px; font:.6rem var(--mono); white-space:normal; }.formula-duo p { color:var(--muted); font-size:.56rem; }
|
||||
.topology-compare { display:grid; grid-template-columns:1fr 1fr; gap:12px; }.topology-compare article { padding:25px; border:1px solid var(--line-strong); background:var(--paper-raised); }.topology-compare code { display:block; margin:23px 0; font:.63rem var(--mono); }.topology-compare p { color:var(--muted); font-size:.58rem; line-height:1.65; }
|
||||
.block-glyph { display:flex; align-items:center; gap:9px; min-height:95px; padding:15px; color:#fff; background:#172437; }.block-glyph > * { min-width:42px; padding:10px; text-align:center; font-style:normal; }.block-glyph i,.block-glyph strong { border:1px solid #5a6c80; font:.5rem var(--mono); }.block-glyph em { color:#d49a68; }
|
||||
.deep-stack-timeline { display:grid; grid-template-columns:repeat(4,1fr); }.deep-stack-timeline article { min-height:205px; padding:20px; border:1px solid var(--line); border-right:0; border-bottom:0; }.deep-stack-timeline article:nth-child(4n) { border-right:1px solid var(--line); }.deep-stack-timeline article:nth-last-child(-n+4) { border-bottom:1px solid var(--line); }.deep-stack-timeline span { color:var(--copper); font:.5rem var(--mono); }.deep-stack-timeline h3 { margin:20px 0 8px; font:650 .85rem var(--serif); }.deep-stack-timeline b { font:.5rem var(--mono); }.deep-stack-timeline p { color:var(--muted); font-size:.53rem; line-height:1.55; }
|
||||
.residual-equation { padding:28px; color:#fff; background:#172437; }.residual-equation > span { color:#d49a68; font:.5rem var(--mono); }.residual-equation > code { display:block; margin:25px 0; font:.65rem var(--mono); white-space:normal; }.residual-equation > p { color:#aebac7; font-size:.56rem; line-height:1.65; }
|
||||
.residual-bars { height:150px; display:flex; align-items:end; gap:8px; border-bottom:1px solid #586a7d; }.residual-bars i { flex:1; height:var(--h); position:relative; background:linear-gradient(#d49a68,#5c6f83); }.residual-bars small { position:absolute; bottom:-18px; color:#8ea0b1; font:8px var(--mono); }
|
||||
.three-questions { display:grid; grid-template-columns:repeat(3,1fr); gap:10px; margin-top:25px; }.three-questions article { padding:20px; border:1px solid var(--line); }.three-questions b { display:block; margin:20px 0 9px; font:650 .78rem var(--serif); }.three-questions p { color:var(--muted); font-size:.55rem; }
|
||||
.hc-diagram { display:grid; grid-template-columns:1.3fr .7fr 1fr .7fr .8fr; gap:10px; align-items:center; }.stream-bank,.mapping,.layer-f,.mix-matrix { min-height:190px; padding:20px; border:1px solid var(--line); background:var(--paper-raised); }.stream-bank i { display:block; margin-top:9px; padding:7px; background:#172437; color:#cbd4dc; font:9px var(--mono); font-style:normal; }.mapping,.layer-f,.mix-matrix { display:flex; flex-direction:column; justify-content:center; text-align:center; }.hc-diagram b { font:650 1rem var(--serif); }.hc-diagram small { margin-top:10px; color:var(--muted); font-size:.5rem; }
|
||||
.mhc-stage { display:grid; grid-template-columns:1fr .75fr 1fr; gap:10px; align-items:center; }.raw-matrix,.stable-matrix { padding:20px; border:1px solid var(--line); }.raw-matrix > span,.stable-matrix > span { color:var(--copper); font:.5rem var(--mono); }.raw-matrix > div,.stable-matrix > div { display:grid; grid-template-columns:repeat(4,1fr); gap:4px; margin:20px 0; }.raw-matrix i,.stable-matrix i { display:grid; place-items:center; aspect-ratio:1; background:rgba(193,124,68,var(--v)); color:var(--ink); font:8px var(--mono); font-style:normal; }.raw-matrix b,.stable-matrix b { font:.52rem var(--mono); }.sinkhorn { padding:22px; color:#fff; background:#172437; text-align:center; }.sinkhorn b,.sinkhorn i,.sinkhorn small { display:block; margin-top:14px; }.sinkhorn b { font:.56rem var(--mono); }.sinkhorn i { color:#d49a68; font:1rem var(--serif); }.sinkhorn small { color:#aebac7; font-size:.48rem; }
|
||||
.proof-cards { display:grid; grid-template-columns:repeat(3,1fr); gap:10px; margin-top:18px; }.proof-cards article { padding:20px; border:1px solid var(--line); }.proof-cards b { display:block; margin:20px 0 9px; font:650 .82rem var(--serif); }.proof-cards p { color:var(--muted); font-size:.54rem; line-height:1.55; }
|
||||
.attnres-diagram { display:grid; grid-template-columns:1fr 1.1fr .8fr; gap:12px; align-items:center; }.history-layers { display:flex; flex-direction:column; gap:6px; }.history-layers div { display:flex; justify-content:space-between; padding:12px; border:1px solid var(--line); background:var(--paper-raised); }.history-layers b { font:.58rem var(--mono); }.depth-softmax { padding:20px; color:#fff; background:#172437; }.depth-softmax > div { display:flex; flex-direction:column; gap:6px; margin:20px 0; }.depth-softmax i { width:var(--w); padding:6px; background:#d49a68; color:#172437; font:8px var(--mono); font-style:normal; }.depth-softmax code { color:#d6dde4; font:.52rem var(--mono); white-space:normal; }.selected-state { padding:22px; border:1px solid var(--line-strong); }.selected-state b { display:block; margin:24px 0 9px; font:650 .9rem var(--serif); }.selected-state p { color:var(--muted); font-size:.54rem; }
|
||||
.full-block-compare { display:grid; grid-template-columns:1fr 1fr; gap:10px; margin-top:20px; }.full-block-compare article { padding:22px; border:1px solid var(--line); }.full-block-compare b { display:block; margin:20px 0 7px; font:650 .85rem var(--serif); }.full-block-compare code { color:var(--copper); font:.56rem var(--mono); }.full-block-compare p { color:var(--muted); font-size:.55rem; line-height:1.55; }
|
||||
.k3-depth-map { display:grid; grid-template-columns:repeat(9,1fr); gap:5px; }.k3-depth-map > * { min-width:0; min-height:225px; padding:11px; border:1px solid var(--line); background:var(--paper-raised); }.k3-depth-map .embedding-source { color:#fff; background:#172437; }.k3-depth-map span { font-size:.4rem; }.k3-depth-map b { display:block; min-height:38px; margin:18px 0 8px; font:650 .58rem var(--serif); }.k3-depth-map article > div { display:grid; grid-template-columns:repeat(2,1fr); gap:3px; }.k3-depth-map i { aspect-ratio:1; background:#677b8e; }.k3-depth-map .partial i { background:#d49a68; }.k3-depth-map small { display:block; margin-top:8px; color:var(--muted); font:7px var(--mono); }.embedding-source small { color:#aebac7; }
|
||||
.correction-note { display:grid; grid-template-columns:220px 1fr; gap:20px; margin-top:18px; padding:20px; border-left:3px solid var(--copper); background:rgba(193,124,68,.08); }.correction-note b { font:650 .72rem var(--serif); }.correction-note p { margin:0; color:var(--muted); font-size:.58rem; line-height:1.65; }
|
||||
.glu-anatomy { display:grid; grid-template-columns:1fr 50px 1fr 50px 1.2fr; align-items:center; margin-top:20px; }.glu-anatomy > div { min-height:155px; padding:19px; border:1px solid var(--line); }.glu-anatomy > strong { color:var(--copper); text-align:center; }.glu-anatomy b,.glu-anatomy i { display:block; margin-top:18px; font:650 .7rem var(--serif); }.glu-anatomy i { color:var(--muted); font-size:.52rem; font-style:normal; }.glu-anatomy .danger { color:#fff; background:#172437; }
|
||||
.outlier-compare { display:grid; grid-template-columns:1fr 1fr; gap:12px; }.outlier-compare article { padding:25px; color:#fff; background:#172437; }.outlier-compare article.kimi-card { background:#263a50; }.outlier-compare h3 { margin:22px 0; font:650 1.15rem var(--serif); }.outlier-compare code { display:block; min-height:70px; color:#dce3e9; font:.6rem/1.8 var(--mono); }.outlier-compare p { color:#b3bfca; font-size:.58rem; line-height:1.7; }.clip-curve,.smooth-curve { position:relative; height:100px; margin:18px 0; border-bottom:1px solid #607286; }.clip-curve i,.smooth-curve i { position:absolute; inset:15px 8% 0; border-top:3px solid #d49a68; border-left:3px solid #d49a68; transform:skewX(-35deg); }.smooth-curve i { border:0; border-radius:50% 50% 0 0; box-shadow:inset 0 3px #d49a68; transform:none; }.gradient-note { margin-top:18px; padding:20px; border:1px solid var(--line); }.gradient-note p { margin:10px 0 0; color:var(--muted); font-size:.58rem; line-height:1.65; }
|
||||
.deepseek-ledger { display:grid; grid-template-columns:repeat(3,1fr); }.deepseek-ledger article { min-height:200px; padding:21px; border:1px solid var(--line); border-right:0; border-bottom:0; }.deepseek-ledger article:nth-child(3n) { border-right:1px solid var(--line); }.deepseek-ledger article:nth-last-child(-n+3) { border-bottom:1px solid var(--line); }.deepseek-ledger h3 { margin:24px 0 10px; font:650 .82rem var(--serif); }.deepseek-ledger p { color:var(--muted); font-size:.55rem; line-height:1.6; }
|
||||
.v4-control-room { position:relative; min-height:440px; margin-top:25px; background:#172437; }.v4-center { position:absolute; inset:50% auto auto 50%; width:260px; padding:25px; border:1px solid #63758a; color:#fff; text-align:center; transform:translate(-50%,-50%); }.v4-center b { display:block; margin-top:18px; font:650 1rem var(--serif); }.v4-control-room article { position:absolute; left:var(--x); top:var(--y); width:26%; min-height:105px; padding:16px; border:1px solid #53667a; color:#fff; background:#25394f; }.v4-control-room article b { display:block; margin:15px 0 6px; font:650 .7rem var(--serif); }.v4-control-room article p { margin:0; color:#aebac7; font-size:.49rem; }
|
||||
.k3-flow { display:grid; grid-template-columns:1fr 30px 1fr 30px 1fr 30px 1fr; align-items:stretch; }.k3-flow article { min-height:240px; padding:21px; border:1px solid var(--line); background:var(--paper-raised); }.k3-flow > i { display:grid; place-items:center; color:var(--copper); font-style:normal; }.k3-flow b { display:block; margin:28px 0 10px; font:650 .82rem var(--serif); }.k3-flow p { color:var(--muted); font-size:.54rem; line-height:1.6; }
|
||||
.k3-nope-evidence { margin-top:20px; padding:25px; color:#fff; background:#172437; }.k3-nope-evidence h3 { margin:20px 0; font:650 1.15rem var(--serif); }.k3-nope-evidence > div { display:flex; align-items:center; gap:12px; }.k3-nope-evidence > div b { min-width:55px; padding:9px; border:1px solid #5c6e81; text-align:center; font:.58rem var(--mono); }.k3-nope-evidence > div i { flex:1; height:2px; background:#d49a68; }.k3-nope-evidence p { color:#b1bdc8; font-size:.58rem; line-height:1.65; }
|
||||
.audit-grid { display:grid; grid-template-columns:repeat(3,1fr); }.audit-grid article { min-height:170px; padding:20px; border:1px solid var(--line); border-right:0; border-bottom:0; }.audit-grid article:nth-child(3n) { border-right:1px solid var(--line); }.audit-grid article:nth-last-child(-n+3) { border-bottom:1px solid var(--line); }.audit-grid span { color:var(--copper); font:.5rem var(--mono); }.audit-grid h3 { margin:20px 0 8px; font:650 .8rem var(--serif); }.audit-grid p { color:var(--muted); font-size:.54rem; line-height:1.55; }
|
||||
.final-thesis { margin-top:28px; padding:30px; color:#fff; background:#172437; }.final-thesis h3 { max-width:980px; margin:25px 0 14px; font:650 clamp(1.3rem,3vw,2.2rem)/1.25 var(--serif); }.final-thesis p { color:#aebac7; font-size:.62rem; line-height:1.7; }
|
||||
.paper-chain { display:grid; grid-template-columns:repeat(3,1fr); }.paper-chain a { position:relative; min-height:190px; padding:20px; border:1px solid var(--line); border-right:0; border-bottom:0; color:var(--ink); text-decoration:none; background:var(--paper-raised); }.paper-chain a:nth-child(3n) { border-right:1px solid var(--line); }.paper-chain a:nth-last-child(-n+3) { border-bottom:1px solid var(--line); }.paper-chain a:hover { background:rgba(193,124,68,.08); }.paper-chain span { color:var(--copper); font:.49rem var(--mono); }.paper-chain h3 { margin:22px 22px 10px 0; font:650 .82rem var(--serif); }.paper-chain p { color:var(--muted); font-size:.53rem; line-height:1.55; }.paper-chain a > b { position:absolute; right:16px; top:16px; color:var(--copper); }
|
||||
@media (max-width: 1100px) {
|
||||
.chapter-hero { grid-template-columns:1fr; }.chapter-hero-aside { border-left:0; border-top:1px solid #53667b; }
|
||||
.article-shell { grid-template-columns:1fr; }.side-rail { position:relative; height:auto; border-right:0; border-bottom:1px solid var(--line); }.side-rail nav { display:grid; grid-template-columns:repeat(3,1fr); }
|
||||
.rep-ledgers,.deep-stack-timeline { grid-template-columns:repeat(2,1fr); }.rep-ledgers article:nth-child(2n),.deep-stack-timeline article:nth-child(2n) { border-right:1px solid var(--line); }.rep-ledgers article:nth-child(4n+1),.deep-stack-timeline article:nth-child(4n+1) { border-right:0; }
|
||||
.four-axis,.solution-grid { grid-template-columns:repeat(2,1fr); }.position-table { grid-template-columns:repeat(2,1fr); }.position-table article { border-right:1px solid var(--line); }
|
||||
.k3-depth-map { grid-template-columns:repeat(3,1fr); }.paper-chain { grid-template-columns:repeat(2,1fr); }.paper-chain a:nth-child(3n) { border-right:0; }.paper-chain a:nth-child(2n) { border-right:1px solid var(--line); }
|
||||
}
|
||||
@media (max-width: 720px) {
|
||||
.chapter-hero-main { padding:58px 22px; }.chapter-hero h1 { font-size:clamp(2.6rem,15vw,4.4rem); }.article { padding-inline:18px; }.article-section { padding:58px 0; }
|
||||
.side-rail nav { grid-template-columns:repeat(2,1fr); }
|
||||
.rep-ledgers,.four-axis,.tradeoff-triangle,.unit-ladder,.wave-timeline,.two-col,.solution-grid,.formula-duo,.topology-compare,.deep-stack-timeline,.three-questions,.proof-cards,.full-block-compare,.outlier-compare,.deepseek-ledger,.audit-grid,.paper-chain { grid-template-columns:1fr; }
|
||||
.rep-ledgers article,.deep-stack-timeline article,.deepseek-ledger article,.audit-grid article,.paper-chain a { border-right:1px solid var(--line)!important; border-bottom:0; }.rep-ledgers article:last-child,.deep-stack-timeline article:last-child,.deepseek-ledger article:last-child,.audit-grid article:last-child,.paper-chain a:last-child { border-bottom:1px solid var(--line); }
|
||||
.object-stack article { grid-template-columns:40px 1fr; }.object-stack p { grid-column:2; }
|
||||
.evidence-note { grid-template-columns:1fr; }.tying-diagram,.context-story,.absolute-grid,.deepseek-position,.nope-layers,.k3-flow { grid-template-columns:1fr; }.tying-diagram > i,.context-story > i,.deepseek-position > i,.k3-flow > i { transform:rotate(90deg); }
|
||||
.table-row,.norm-row,.activation-row { grid-template-columns:1fr; }.table-row > *,.norm-row > *,.activation-row > * { border-right:0; }
|
||||
.attention-ffn,.attnres-diagram { grid-template-columns:1fr; }.attention-ffn > div { padding:15px; }.position-table { grid-template-columns:1fr; }
|
||||
.rope-stage { grid-template-columns:1fr; }.rope-stage > div { border-left:0; border-top:1px solid #485b70; }.extrapolation-map { grid-template-columns:1fr; }.risk-pins { display:none; }
|
||||
.mla-decoupling { grid-template-columns:1fr; }.hc-diagram,.mhc-stage { grid-template-columns:1fr; }.correction-note { grid-template-columns:1fr; }
|
||||
.glu-anatomy { grid-template-columns:1fr; }.glu-anatomy > strong { padding:12px; }.v4-control-room { min-height:auto; padding:16px; }.v4-center,.v4-control-room article { position:static; width:auto; margin:8px; transform:none; }
|
||||
.k3-nope-evidence > div { flex-wrap:wrap; }.k3-nope-evidence > div i { min-width:20px; }.paper-chain a:nth-last-child(-n+3) { border-bottom:0; }
|
||||
}
|
||||
</style>
|
||||
+29
-1
@@ -8,6 +8,7 @@ const routes: Record<string, string> = {
|
||||
roadmap: "/roadmap/",
|
||||
"foundations/language-models": "/foundations/language-models/",
|
||||
foundations: "/foundations/",
|
||||
"architecture/representation": "/architecture/representation/",
|
||||
scaling: "/scaling/",
|
||||
"pretraining/data": "/pretraining/data/",
|
||||
moe: "/moe/",
|
||||
@@ -87,6 +88,7 @@ const paths = [
|
||||
<a class="button primary" href="/roadmap/">选择学习路径 <span aria-hidden="true">↓</span></a>
|
||||
<a class="button" href="/foundations/language-models/">语言模型从哪里来</a>
|
||||
<a class="button" href="/foundations/">注意力与 Transformer</a>
|
||||
<a class="button" href="/architecture/representation/">表示、位置与残差</a>
|
||||
<a class="button" href="/k3/">直接解剖 K3</a>
|
||||
<a class="button" href="/deepseek/">DeepSeek 专题</a>
|
||||
<a class="button" href="/scaling/">Scaling Laws 专题</a>
|
||||
@@ -110,7 +112,7 @@ const paths = [
|
||||
<div class="hero-stats">
|
||||
<div><b>17</b><span>核心专题</span></div>
|
||||
<div><b>151</b><span>K3 报告来源</span></div>
|
||||
<div><b>450</b><span>关键论文索引</span></div>
|
||||
<div><b>480</b><span>关键论文索引</span></div>
|
||||
<div><b>47p</b><span>K3 技术报告</span></div>
|
||||
</div>
|
||||
</aside>
|
||||
@@ -124,6 +126,22 @@ const paths = [
|
||||
|
||||
<section class="section compact release-section" id="new-chapters">
|
||||
<div class="release-grid">
|
||||
<a class="release-card representation-release" href="/architecture/representation/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / CHAPTER 03</span> TOKEN · POSITION · DEPTH · FFN</p>
|
||||
<h2>一条 hidden state 同时沿词表、位置、深度和非线性四个坐标轴被改写</h2>
|
||||
<p>
|
||||
用二十张问题账从 Token、embedding 与上下文化表示，走到 RoPE / NoPE、Pre / Post-LN、
|
||||
DeepSeek mHC 与 partial RoPE，以及 Kimi K3 的 9 个 AttnRes 深度来源和有界 SiTU-GLU。
|
||||
</p>
|
||||
</div>
|
||||
<dl>
|
||||
<div><dt>LINEAGE</dt><dd>2003 → 2026</dd></div>
|
||||
<div><dt>NODES</dt><dd>66 个一手节点</dd></div>
|
||||
<div><dt>LAB</dt><dd>Token · 位置 · Norm · Residual / FFN</dd></div>
|
||||
</dl>
|
||||
<span class="release-arrow" aria-hidden="true">从一个 Token 进入完整表示生命史 →</span>
|
||||
</a>
|
||||
<a class="release-card evaluation-release" href="/evaluation/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / CHAPTER 15</span> SCORE · PROTOCOL · THREAT MODEL</p>
|
||||
@@ -581,6 +599,7 @@ const paths = [
|
||||
transition: transform 180ms ease, border-color 180ms ease;
|
||||
}
|
||||
|
||||
.representation-release,
|
||||
.inference-release,
|
||||
.agent-release,
|
||||
.alignment-release,
|
||||
@@ -595,6 +614,14 @@ const paths = [
|
||||
min-height: 510px;
|
||||
}
|
||||
|
||||
.representation-release {
|
||||
background:
|
||||
radial-gradient(circle at 82% 18%, rgba(159, 91, 52, 0.22), transparent 31%),
|
||||
radial-gradient(circle at 61% 74%, rgba(56, 91, 128, 0.16), transparent 28%),
|
||||
repeating-linear-gradient(90deg, transparent 0 56px, rgba(159, 91, 52, 0.04) 56px 57px),
|
||||
var(--paper-raised);
|
||||
}
|
||||
|
||||
.inference-release {
|
||||
background:
|
||||
radial-gradient(circle at 82% 18%, rgba(35, 86, 84, 0.22), transparent 31%),
|
||||
@@ -733,6 +760,7 @@ const paths = [
|
||||
padding-bottom: 76px;
|
||||
}
|
||||
|
||||
.representation-release,
|
||||
.inference-release,
|
||||
.alignment-release,
|
||||
.transformer-release,
|
||||
|
||||
@@ -12,6 +12,7 @@ const workstreams = [
|
||||
{ label: "Kimi K3 深读", value: 66, next: "扩写 pre-training / infra 逐图笔记" },
|
||||
{ label: "语言模型前史", value: 78, next: "逐图精读 Kneser–Ney、LSTM 与 Bahdanau，并加入真实小语料复现" },
|
||||
{ label: "Transformer 基础", value: 79, next: "逐图精读多头电路、Pre/Post-LN 与真实 kernel / KV 配置" },
|
||||
{ label: "表示、位置与残差高速公路", value: 81, next: "加入真实 hidden-state / norm traces、长上下文位置外推复现与更多深层稳定性消融" },
|
||||
{ label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" },
|
||||
{ label: "数据工程与预训练配方", value: 73, next: "逐图精读 FineWeb / DCLM，加入真实去重与 mixture traces" },
|
||||
{ label: "DeepSeek 专题", value: 71, next: "补 R1 / DAPO 的逐图训练轨迹与复现对照" },
|
||||
@@ -49,7 +50,7 @@ const workstreams = [
|
||||
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
|
||||
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
|
||||
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 09:24 CST</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 10:26 CST</dd></div>
|
||||
<div><dt>MODE</dt><dd>持续迭代，不锁死版本</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
@@ -59,7 +60,7 @@ const workstreams = [
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<p class="eyebrow"><span>01</span> WORKSTREAMS</p>
|
||||
<h2>二十条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
|
||||
<h2>二十一条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
|
||||
</div>
|
||||
<p class="section-lead">
|
||||
内容首版优先打通全局脉络；随后每轮迭代选择一个专题推进到论文/工程层，并做独立事实复核。
|
||||
@@ -96,10 +97,11 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
|
||||
<article><span>✓</span><h3>17 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
|
||||
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
|
||||
<article><span>✓</span><h3>五十一个原创交互视图</h3><p>K3、语言模型前史、Transformer、DeepSeek、长上下文、MoE、推理、Agent、多模态，以及训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>十六篇首版长文</h3><p>K3、语言模型前史、Transformer、DeepSeek、Scaling、数据工程、长上下文、MoE、后训练、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>五十五个原创交互视图</h3><p>K3、语言模型前史、Transformer、表示深度、DeepSeek、长上下文、MoE、推理、Agent、多模态，以及训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>十七篇首版长文</h3><p>K3、语言模型前史、Transformer、表示/位置/残差、DeepSeek、Scaling、数据工程、长上下文、MoE、后训练、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>语言模型前史深度专题</h3><p>八张独立问题账、33 个正式节点、20 段长文与概率—向量—记忆—对齐四联实验。</p></article>
|
||||
<article><span>✓</span><h3>Transformer 深度专题</h3><p>十张独立问题账、40 个正式节点、21 段正文与 QKV—Mask—多头位置—Block 成本四联实验。</p></article>
|
||||
<article><span>✓</span><h3>表示、位置与残差高速公路深度专题</h3><p>二十张问题账、66 个一手节点、DeepSeek/Kimi 双谱系，以及 Token—位置—Norm—Residual/FFN 四联实验。</p></article>
|
||||
<article><span>✓</span><h3>Scaling Laws 深度专题</h3><p>九张账、29 个一手节点、DeepSeek/Kimi 双谱系与曲面—部署—复用—涌现四联实验。</p></article>
|
||||
<article><span>✓</span><h3>数据工程深度专题</h3><p>十二张账、31 个一手节点、DeepSeek/Kimi 双谱系与流水线—去重—混合—改写四联实验。</p></article>
|
||||
<article><span>✓</span><h3>长上下文深度专题</h3><p>五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。</p></article>
|
||||
@@ -112,7 +114,7 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>原生多模态深度专题</h3><p>十六张账、55 个一手节点、DeepSeek 三分支、Kimi 三代 MoonViT，以及 Token—连接器—光学压缩—视觉闭环四联实验。</p></article>
|
||||
<article><span>✓</span><h3>推理服务与低成本部署深度专题</h3><p>十八本账、62 个一手节点、DeepSeek V2→V4 与 Mooncake→K3 双谱系，以及显存—阶段—推测—集群四联实验。</p></article>
|
||||
<article><span>✓</span><h3>评测、安全与“到底强不强”深度专题</h3><p>二十二张账、80 个一手节点、DeepSeek/K3 评测协议谱系，以及指标—Judge—污染—系统安全四联实验。</p></article>
|
||||
<article><span>✓</span><h3>450 篇关键论文索引</h3><p>新增 BLEU、GLUE、HumanEval、MT-Bench、LiveBench、HarmBench、InjecAgent、AILuminate 等 50 个评测安全节点。</p></article>
|
||||
<article><span>✓</span><h3>480 篇关键论文索引</h3><p>新增 output embedding、ELMo、BLT、Fixup、ReZero、Hyper-Connections、mHC、xPos、FIRE、LongRoPE 等 30 个表示与深度节点。</p></article>
|
||||
<article><span>✓</span><h3>公开仓库与自托管发布</h3><p>源码公开到 git.k1412.top，网站由不可变镜像、Compose Manager 与 HTTPS 交付。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
@@ -128,6 +130,7 @@ const workstreams = [
|
||||
<div class="queue-table">
|
||||
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
|
||||
<div><span>P0</span><strong>Transformer 二轮</strong><p>多头电路逐图 → Pre/Post-LN 真实 traces → Flash/KV 配置与 kernel 对照</p><em>逐图笔记 + 实测边界</em></div>
|
||||
<div><span>P0</span><strong>表示、位置与残差二轮</strong><p>真实 hidden-state / norm traces → 长上下文位置外推 → mHC / AttnRes 深层稳定性消融</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P0</span><strong>语言模型前史二轮</strong><p>Kneser–Ney / LSTM / Bahdanau 逐图 → 真实小语料复现 → tokenizer 公平性</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P0</span><strong>Scaling Laws 二轮</strong><p>真实拟合复现 → 置信区间 → 更多模型族与下游任务外推</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P1</span><strong>数据工程二轮</strong><p>FineWeb / DCLM 逐图 → 真实去重误伤 → mixture traces 与污染案例</p><em>逐图笔记 + 案例库</em></div>
|
||||
@@ -197,6 +200,8 @@ const workstreams = [
|
||||
<div><time>2026-07-29</time><b>光学压缩实验分开报告值、教学插值与证据外区域</b><p>DeepSeek-OCR 的 <10× / 20× 锚点标成作者报告；中间只做显式教学插值，超过范围不外推。</p></div>
|
||||
<div><time>2026-07-29</time><b>推理服务按十八本账组织</b><p>权重、增长状态、分配、阶段、batch、cache、kernel、推测、网络、路由、故障与经济性不再压成单一 tokens/s。</p></div>
|
||||
<div><time>2026-07-29</time><b>DeepSeek 与 Kimi 服务谱系按状态对象重建</b><p>MLA→V4 异构状态与 Mooncake→KDA→K3 混合缓存分开说明；作者报告、精确公式和教学估算使用不同标签。</p></div>
|
||||
<div><time>2026-07-29</time><b>表示与深度按二十张账组织</b><p>计算单位、词表接口、上下文化、位置、外推、Norm 对象、拓扑、残差路由与非线性极值不再混成一个 hidden-state 名词。</p></div>
|
||||
<div><time>2026-07-29</time><b>K3 Block AttnRes 来源数按原报告重算</b><p>93 层按 12 层形成 8 个 layer blocks（7 个完整块加 1 个尾块）；再加 embedding，共 9 个 block-level 来源，废弃早期错误的“2 层一块”读法。</p></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ const stages = [
|
||||
{stage.items.map((number) => {
|
||||
const chapter = chapters.find((item) => item.number === number)!;
|
||||
return (
|
||||
<a href={`#chapter-${number}`} class:list={{ hot: ["02", "06", "07", "11"].includes(number) }}>
|
||||
<a href={`#chapter-${number}`} class:list={{ hot: ["02", "03", "06", "07", "11"].includes(number) }}>
|
||||
<b>{number}</b>
|
||||
<p>{chapter.title}</p>
|
||||
</a>
|
||||
|
||||
Reference in New Issue
Block a user