wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: publish data engineering chapter

Browse Source
This commit is contained in:
wuyang
2026-07-29 03:07:40 +08:00
parent 229deecb00
commit ee0361b3de
19 changed files with 5824 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data/chapters.ts
+5 -5
View File
@@ -86,12 +86,12 @@ export const chapters: Chapter[] = [
title: "数据工程与预训练配方",
kicker: "DATA",
question: "更多网页为什么不等于更好的模型？",
summary: "追踪采集、过滤、去重、混合、课程、合成数据与污染控制，并明确公开报告的知识边界。",
status: "queued",
progress: 8,
papers: 14,
summary: "用十二张账追踪采集、解析、过滤、去重、混合、改写、Tokenizer、Packing、课程与污染，并串起 DeepSeek / Kimi 数据谱系。",
status: "published",
progress: 73,
papers: 31,
prerequisites: ["01", "04"],
highlights: ["数据质量", "混合策略", "污染"],
highlights: ["十二张账", "DeepSeek/Kimi 谱系", "四联实验"],
},
{
number: "06",
src/data/papers.ts
+182 -20
View File
@@ -4,6 +4,7 @@ export type PaperTopic =
| "长上下文"
| "MoE"
| "Scaling"
| "数据"
| "训练系统"
| "低精度"
| "后训练"
@@ -123,8 +124,8 @@ export const papers: Paper[] = [
year: 2019,
title: "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer",
url: "https://arxiv.org/abs/1910.10683",
topics: ["Transformer", "Scaling"],
contribution: "T5 统一 text-to-text 接口并系统研究架构与数据。",
topics: ["Transformer", "Scaling", "数据"],
contribution: "T5 统一 text-to-text 接口，并以 C4 建立现代 Common Crawl 清洗基线。",
verified: true,
},
{
@@ -397,7 +398,7 @@ export const papers: Paper[] = [
year: 2021,
title: "GLaM: Efficient Scaling of Language Models with Mixture-of-Experts",
url: "https://arxiv.org/abs/2112.06905",
topics: ["MoE", "Scaling"],
topics: ["MoE", "Scaling", "数据"],
contribution: "展示 MoE 在能耗与推理计算上高效扩展语言模型。",
verified: true,
},
@@ -446,8 +447,8 @@ export const papers: Paper[] = [
year: 2024,
title: "DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model",
url: "https://arxiv.org/abs/2405.04434",
topics: ["MoE", "长上下文", "训练系统"],
contribution: "联合 DeepSeekMoE 与 MLA，直接压缩推理 KV Cache。",
topics: ["MoE", "长上下文", "数据", "训练系统"],
contribution: "联合 DeepSeekMoE 与 MLA，并披露 8.1T 中英数据、过滤取舍和训练配方。",
spotlight: "DeepSeek",
verified: true,
},
@@ -472,7 +473,7 @@ export const papers: Paper[] = [
year: 2024,
title: "DeepSeek-V3 Technical Report",
url: "https://arxiv.org/abs/2412.19437",
topics: ["MoE", "Scaling", "训练系统", "低精度"],
topics: ["MoE", "Scaling", "数据", "训练系统", "低精度"],
contribution: "671B-A37B；FP8、MTP、无辅助损失平衡与 DualPipe。",
spotlight: "DeepSeek",
verified: true,
@@ -545,8 +546,8 @@ export const papers: Paper[] = [
year: 2024,
title: "DeepSeek LLM: Scaling Open-Source Language Models with Longtermism",
url: "https://arxiv.org/abs/2401.02954",
topics: ["Scaling"],
contribution: "中英 dense 基线与公开 scaling-law 研究。",
topics: ["Scaling", "数据"],
contribution: "中英 dense 基线、跨 91 个 Common Crawl dump 去重与公开 scaling-law 研究。",
spotlight: "DeepSeek",
verified: true,
},
@@ -554,7 +555,7 @@ export const papers: Paper[] = [
year: 2021,
title: "The Pile: An 800GB Dataset of Diverse Text for Language Modeling",
url: "https://arxiv.org/abs/2101.00027",
topics: ["Scaling"],
topics: ["Scaling", "数据"],
contribution: "公开多域预训练语料与数据配比研究基础。",
verified: true,
},
@@ -562,7 +563,7 @@ export const papers: Paper[] = [
year: 2023,
title: "The RefinedWeb Dataset for Falcon LLM",
url: "https://arxiv.org/abs/2306.01116",
topics: ["Scaling"],
topics: ["Scaling", "数据"],
contribution: "证明高质量过滤与去重的 Web 数据可以支撑强模型。",
verified: true,
},
@@ -570,10 +571,170 @@ export const papers: Paper[] = [
year: 2024,
title: "Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research",
url: "https://arxiv.org/abs/2402.00159",
topics: ["Scaling"],
topics: ["Scaling", "数据"],
contribution: "开放 3T Token 语料与完整数据处理工具。",
verified: true,
},
{
year: 2019,
title: "CCNet: Extracting High Quality Monolingual Datasets from Web Crawl Data",
url: "https://arxiv.org/abs/1911.00359",
topics: ["数据"],
contribution: "把去重、语言识别和 Wikipedia-like 质量过滤组织成可扩展网页语料流水线。",
verified: true,
},
{
year: 2020,
title: "Extracting Training Data from Large Language Models",
url: "https://arxiv.org/abs/2012.07805",
topics: ["数据", "评测"],
contribution: "从 GPT-2 抽取逐字训练片段与 PII，实证训练记忆和隐私泄露风险。",
verified: true,
},
{
year: 2021,
title: "Deduplicating Training Data Makes Language Models Better",
url: "https://arxiv.org/abs/2107.06499",
topics: ["数据", "评测"],
contribution: "连接近重复、验证集重合、训练效率与逐字记忆，建立现代 fuzzy dedup 主线。",
verified: true,
},
{
year: 2023,
title: "ROOTS: One Massive Multilingual Dataset for 1.6TB of Text",
url: "https://arxiv.org/abs/2303.03915",
topics: ["数据"],
contribution: "总结 59 种语言、1.6TB 多语言语料的协作建设、治理与数据文档。",
verified: true,
},
{
year: 2022,
title: "Efficient Training of Language Models to Fill in the Middle",
url: "https://arxiv.org/abs/2207.14255",
topics: ["数据"],
contribution: "通过 PSM/SPM 数据重排获得代码中间补全能力，并消融变换率和格式。",
verified: true,
},
{
year: 2023,
title: "DoReMi: Optimizing Data Mixtures Speeds Up Language Model Pretraining",
url: "https://arxiv.org/abs/2305.10429",
topics: ["数据", "Scaling"],
contribution: "用小代理模型学习 domain weights，再迁移到更大目标模型。",
verified: true,
},
{
year: 2023,
title: "SemDeDup: Data-efficient Learning at Web-scale through Semantic Deduplication",
url: "https://arxiv.org/abs/2303.09540",
topics: ["数据"],
contribution: "在预训练 embedding 空间聚类语义近重复，研究硬删除与多样性收益。",
verified: true,
},
{
year: 2023,
title: "D4: Improving LLM Pretraining via Document De-Duplication and Diversification",
url: "https://arxiv.org/abs/2308.12284",
topics: ["数据"],
contribution: "先去重再主动保留多样性，说明智能重复与随机重复并不等价。",
verified: true,
},
{
year: 2023,
title: "PALOMA: A Benchmark for Evaluating Language Model Fit",
url: "https://arxiv.org/abs/2312.10523",
topics: ["数据", "评测"],
contribution: "提出固定训练顺序、子文档去污染、细粒度 domain 和 bits-per-byte 语料评测协议。",
verified: true,
},
{
year: 2024,
title: "The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale",
url: "https://arxiv.org/abs/2406.17557",
topics: ["数据"],
contribution: "以大规模网页清洗消融构建 FineWeb 与教育价值子集。",
verified: true,
},
{
year: 2024,
title: "DataComp-LM: In Search of the Next Generation of Language Model Pretraining Datasets",
url: "https://arxiv.org/abs/2406.11794",
topics: ["数据", "Scaling", "评测"],
contribution: "固定 240T candidate pool、模型尺度与 53 项评测，使数据 pipeline 可以同协议比较。",
verified: true,
},
{
year: 2024,
title: "Rephrasing the Web: A Recipe for Compute and Data-Efficient Language Modeling",
url: "https://arxiv.org/abs/2401.16380",
topics: ["数据"],
contribution: "WRAP 用受控改写增加网页语料的表达覆盖，并研究 Token utility。",
verified: true,
},
{
year: 2024,
title: "Fewer Truncations Improve Language Modeling",
url: "https://arxiv.org/abs/2404.10830",
topics: ["数据", "训练系统"],
contribution: "Best-fit packing 减少长文档截断，在固定序列长度下保护文档完整性。",
verified: true,
},
{
year: 2024,
title: "SoftDedup: Efficient Data Reweighting at Scale",
url: "https://arxiv.org/abs/2407.06654",
topics: ["数据"],
contribution: "按常见 n-gram 频率降低样本权重，探索比硬删除更连续的去重策略。",
verified: true,
},
{
year: 2024,
title: "MATES: Model-Aware Data Selection for Efficient Language Model Pretraining",
url: "https://arxiv.org/abs/2406.06046",
topics: ["数据", "Scaling"],
contribution: "以当前模型状态动态估计样本对目标验证分布的影响。",
verified: true,
},
{
year: 2024,
title: "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
url: "https://arxiv.org/abs/2401.06059",
topics: ["数据", "评测"],
contribution: "区分 text contamination 与 ground-truth contamination，并检查 n-gram 检测的边界。",
verified: true,
},
{
year: 2024,
title: "MM1: Methods, Analysis & Insights from Multimodal LLM Pre-training",
url: "https://arxiv.org/abs/2403.09611",
topics: ["数据", "多模态"],
contribution: "系统消融 caption、interleaved image-text、text-only mixture 与视觉组件。",
verified: true,
},
{
year: 2024,
title: "JEST: Joint Example Selection for Multimodal Learning",
url: "https://arxiv.org/abs/2406.17711",
topics: ["数据", "多模态"],
contribution: "按 batch 的联合可学习性选择多模态样本，研究动态数据价值。",
verified: true,
},
{
year: 2025,
title: "Group-MATES: Data-Efficient Pretraining with Group-Level Data Influence Modeling",
url: "https://arxiv.org/abs/2502.14709",
topics: ["数据", "Scaling"],
contribution: "把模型感知的数据影响估计扩展到 group-level 选择。",
verified: true,
},
{
year: 2025,
title: "Swallow: Continued Pre-training with Synthetic Data for Efficient Language Model Adaptation",
url: "https://arxiv.org/abs/2505.02881",
topics: ["数据", "后训练"],
contribution: "用 SwallowCode 与 SwallowMath 研究受控合成数据的继续预训练收益与边界。",
verified: true,
},
{
year: 2017,
title: "Deep Learning Scaling is Predictable, Empirically",
@@ -1157,8 +1318,8 @@ export const papers: Paper[] = [
year: 2024,
title: "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
url: "https://arxiv.org/abs/2402.03300",
topics: ["推理", "后训练"],
contribution: "数学数据管线与无独立 critic 的 GRPO。",
topics: ["数据", "推理", "后训练"],
contribution: "从 40B HTML pages 发现 120.2B 数学语料，并提出无独立 critic 的 GRPO。",
spotlight: "DeepSeek",
verified: true,
},
@@ -1344,8 +1505,8 @@ export const papers: Paper[] = [
year: 2025,
title: "Kimi K2: Open Agentic Intelligence",
url: "https://arxiv.org/abs/2507.20534",
topics: ["MoE", "训练系统", "Agent", "后训练", "推理"],
contribution: "开放 1T MoE Agent 模型，以 verifiable gym 与 self-critique rubric 做 joint RL。",
topics: ["MoE", "数据", "训练系统", "Agent", "后训练", "推理"],
contribution: "开放 1T MoE Agent 模型，以 15.5T、知识/数学改写与 verifiable gym / self-critique joint RL 协同。",
spotlight: "Kimi",
verified: true,
},
@@ -1353,7 +1514,7 @@ export const papers: Paper[] = [
year: 2026,
title: "Kimi K2.5: Visual Agentic Intelligence",
url: "https://arxiv.org/abs/2602.02276",
topics: ["训练系统", "Agent", "多模态", "推理"],
topics: ["数据", "训练系统", "Agent", "多模态", "推理"],
contribution: "把视觉、工具使用和并行 Agent Swarm 纳入统一后训练。",
spotlight: "Kimi",
verified: true,
@@ -1484,8 +1645,8 @@ export const papers: Paper[] = [
year: 2026,
title: "DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence",
url: "https://arxiv.org/abs/2606.19348",
topics: ["长上下文", "MoE", "训练系统", "推理"],
contribution: "CSA/HCA 混合注意力、mHC、Muon 与 1M 上下文双模型。",
topics: ["长上下文", "MoE", "数据", "训练系统", "推理"],
contribution: "32/33T 长文与 agentic 数据、CSA/HCA、mHC、Muon 和 1M 上下文双模型。",
spotlight: "DeepSeek",
verified: true,
},
@@ -1493,8 +1654,8 @@ export const papers: Paper[] = [
year: 2026,
title: "Kimi K3: Open Frontier Intelligence",
url: "https://arxiv.org/abs/2607.24653",
topics: ["长上下文", "MoE", "训练系统", "低精度", "推理", "Agent", "多模态", "评测"],
contribution: "2.8T-A104B、KDA/MLA、AttnRes、Stable LatentMoE 与 1M Agentic RL。",
topics: ["长上下文", "MoE", "数据", "训练系统", "低精度", "推理", "Agent", "多模态", "评测"],
contribution: "四文本域与视觉语料、2.8T-A104B、KDA/MLA、Stable LatentMoE 与 1M Agentic RL。",
spotlight: "Kimi",
verified: true,
},
@@ -1506,6 +1667,7 @@ export const paperTopics: PaperTopic[] = [
"长上下文",
"MoE",
"Scaling",
"数据",
"训练系统",
"低精度",
"后训练",