feat: publish native multimodal chapter

2026-07-29 07:51:42 +08:00
parent 8e5f08ac21
commit 9f5673213d
24 changed files with 4867 additions and 41 deletions
@@ -2416,6 +2416,22 @@ export const papers: Paper[] = [
    contribution: "面向 Agentic AI 的 Firecracker microVM 环境，支持 pause、resume、fork、snapshot 与高密度运行。",
    verified: true,
  },
+  {
+    year: 2012,
+    title: "ImageNet Classification with Deep Convolutional Neural Networks",
+    url: "https://proceedings.neurips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks",
+    topics: ["多模态"],
+    contribution: "AlexNet 以 GPU、大规模卷积网络与 ImageNet 监督推动可扩展视觉表示。",
+    verified: true,
+  },
+  {
+    year: 2015,
+    title: "Deep Residual Learning for Image Recognition",
+    url: "https://arxiv.org/abs/1512.03385",
+    topics: ["多模态", "Transformer"],
+    contribution: "残差连接让更深视觉骨干稳定优化，成为多模态视觉塔的重要前史。",
+    verified: true,
+  },
  {
    year: 2020,
    title: "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale",
@@ -2424,6 +2440,14 @@ export const papers: Paper[] = [
    contribution: "ViT 把图像 patch 视为 Token 输入 Transformer。",
    verified: true,
  },
+  {
+    year: 2021,
+    title: "Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision",
+    url: "https://arxiv.org/abs/2102.05918",
+    topics: ["多模态", "数据", "Scaling"],
+    contribution: "ALIGN 用超大规模噪声 web 图文对训练双编码器，展示数据规模路线。",
+    verified: true,
+  },
  {
    year: 2021,
    title: "Learning Transferable Visual Models From Natural Language Supervision",
@@ -2432,6 +2456,30 @@ export const papers: Paper[] = [
    contribution: "CLIP 以海量图文对比学习建立可迁移视觉语义空间。",
    verified: true,
  },
+  {
+    year: 2021,
+    title: "Multimodal Few-Shot Learning with Frozen Language Models",
+    url: "https://arxiv.org/abs/2106.13884",
+    topics: ["多模态"],
+    contribution: "Frozen 用视觉编码器生成连续前缀，连接冻结语言模型并激活多模态 in-context learning。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "LiT: Zero-Shot Transfer with Locked-image Text Tuning",
+    url: "https://arxiv.org/abs/2111.07991",
+    topics: ["多模态", "后训练"],
+    contribution: "锁定强图像塔，只训练文本侧去匹配既有视觉空间。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation",
+    url: "https://arxiv.org/abs/2201.12086",
+    topics: ["多模态", "数据"],
+    contribution: "用 captioner 生成描述、filter 清理噪声，同时服务视觉理解与生成。",
+    verified: true,
+  },
  {
    year: 2022,
    title: "Flamingo: a Visual Language Model for Few-Shot Learning",
@@ -2448,6 +2496,134 @@ export const papers: Paper[] = [
    contribution: "用轻量 Q-Former 桥接冻结视觉编码器与 LLM。",
    verified: true,
  },
+  {
+    year: 2023,
+    title: "MiniGPT-4: Enhancing Vision-language Understanding with Advanced Large Language Models",
+    url: "https://arxiv.org/abs/2304.10592",
+    topics: ["多模态", "后训练"],
+    contribution: "用少量高质量视觉对话对齐冻结视觉编码器与冻结语言模型。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning",
+    url: "https://arxiv.org/abs/2305.06500",
+    topics: ["多模态", "后训练"],
+    contribution: "让 Q-Former 感知指令，并系统研究跨任务视觉指令泛化。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "PaLI-X: On Scaling up a Multilingual Vision and Language Model",
+    url: "https://arxiv.org/abs/2305.18565",
+    topics: ["多模态", "Scaling"],
+    contribution: "从组件规模与多语言任务混合两轴扩展视觉—语言模型。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "OCRBench: On the Hidden Mystery of OCR in Large Multimodal Models",
+    url: "https://arxiv.org/abs/2305.07895",
+    topics: ["多模态", "评测"],
+    contribution: "联合评测文字识别、场景文字 VQA、文档 VQA、KIE 与手写公式。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Evaluating Object Hallucination in Large Vision-Language Models",
+    url: "https://arxiv.org/abs/2305.10355",
+    topics: ["多模态", "评测"],
+    contribution: "POPE 用轮询式对象存在性问题更稳定地诊断视觉幻觉。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Kosmos-2: Grounding Multimodal Large Language Models to the World",
+    url: "https://arxiv.org/abs/2306.14824",
+    topics: ["多模态"],
+    contribution: "用 location token 和 grounded image-text pairs 把文本指称绑定到视觉区域。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Patch n' Pack: NaViT, a Vision Transformer for any Aspect Ratio and Resolution",
+    url: "https://arxiv.org/abs/2307.06304",
+    topics: ["多模态", "训练系统"],
+    contribution: "在固定 Token 预算内打包不同分辨率与长宽比图像的 patch 序列。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "MMBench: Is Your Multi-modal Model an All-around Player?",
+    url: "https://arxiv.org/abs/2307.06281",
+    topics: ["多模态", "评测"],
+    contribution: "用双语多选、能力分层与 CircularEval 建立综合 VLM 评测。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond",
+    url: "https://arxiv.org/abs/2308.12966",
+    topics: ["多模态"],
+    contribution: "把开放视觉对话、OCR 与 grounding 能力整合进 Qwen 视觉语言路线。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Improved Baselines with Visual Instruction Tuning",
+    url: "https://arxiv.org/abs/2310.03744",
+    topics: ["多模态", "后训练"],
+    contribution: "LLaVA-1.5 用两层 MLP、更多任务数据与清晰训练配方强化简单基线。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts",
+    url: "https://arxiv.org/abs/2310.02255",
+    topics: ["多模态", "推理", "评测"],
+    contribution: "用图表、几何、函数与视觉数学题联合考察感知和多步推理。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "HallusionBench: An Advanced Diagnostic Suite for Entangled Language Hallucination and Visual Illusion",
+    url: "https://arxiv.org/abs/2310.14566",
+    topics: ["多模态", "评测"],
+    contribution: "用控制组问题区分语言幻觉、视觉错觉、回答倾向与逻辑一致性。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Video-LLaVA: Learning United Visual Representation by Alignment Before Projection",
+    url: "https://arxiv.org/abs/2311.10122",
+    topics: ["多模态"],
+    contribution: "在投影前对齐图像和视频表示，探索共同视觉—语言空间。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI",
+    url: "https://arxiv.org/abs/2311.16502",
+    topics: ["多模态", "推理", "评测"],
+    contribution: "以 11.5K 大学级问题覆盖六大学科、183 子领域与 30 种图像类型。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "LLaMA-VID: An Image is Worth 2 Tokens in Large Language Models",
+    url: "https://arxiv.org/abs/2311.17043",
+    topics: ["多模态", "长上下文"],
+    contribution: "用内容 Token 与上下文 Token 压缩长视频视觉序列。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "RLHF-V: Towards Trustworthy MLLMs via Behavior Alignment from Fine-grained Correctional Human Feedback",
+    url: "https://arxiv.org/abs/2312.00849",
+    topics: ["多模态", "后训练", "评测"],
+    contribution: "以 segment-level 人工纠错和 dense DPO 对齐视觉忠实度。",
+    verified: true,
+  },
  {
    year: 2023,
    title: "Visual Instruction Tuning",
@@ -2464,6 +2640,55 @@ export const papers: Paper[] = [
    contribution: "SigLIP 以成对 sigmoid loss 简化大规模图文对比。",
    verified: true,
  },
+  {
+    year: 2024,
+    title: "DeepSeek-VL: Towards Real-World Vision-Language Understanding",
+    url: "https://arxiv.org/abs/2403.05525",
+    topics: ["多模态", "数据"],
+    contribution: "面向截图、PDF、OCR 与图表，用混合视觉编码器处理高分辨率并管理模态竞争。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "How Far Are We to GPT-4V? Closing the Gap to Commercial Multimodal Models with Open-Source Suites",
+    url: "https://arxiv.org/abs/2404.16821",
+    topics: ["多模态", "Scaling"],
+    contribution: "InternVL 1.5 结合 6B 视觉塔、最多 40 个动态 tile 与高质量双语数据。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Chameleon: Mixed-Modal Early-Fusion Foundation Models",
+    url: "https://arxiv.org/abs/2405.09818",
+    topics: ["多模态", "Transformer"],
+    contribution: "把图像与文字离散成可交错 Token，用 early-fusion 自回归主干统一建模。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Video-MME: The First-Ever Comprehensive Evaluation Benchmark of Multi-modal LLMs in Video Analysis",
+    url: "https://arxiv.org/abs/2405.21075",
+    topics: ["多模态", "评测"],
+    contribution: "用 900 段短中长视频、字幕和音频条件系统评估视频理解。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Visual Sketchpad: Sketching as a Visual Chain of Thought for Multimodal Language Models",
+    url: "https://arxiv.org/abs/2406.09403",
+    topics: ["多模态", "推理", "Agent"],
+    contribution: "让模型画线、框选并调用视觉工具，以新视觉产物继续推理。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "SOLO: A Single Transformer for Scalable Vision-Language Modeling",
+    url: "https://arxiv.org/abs/2407.06438",
+    topics: ["多模态", "Transformer", "Scaling"],
+    contribution: "探索单一 Transformer 视觉—语言架构与可稳定训练的开放配方。",
+    verified: true,
+  },
  {
    year: 2024,
    title: "LLaVA-OneVision: Easy Visual Task Transfer",
@@ -2472,6 +2697,82 @@ export const papers: Paper[] = [
    contribution: "统一单图、多图与视频的视觉指令迁移。",
    verified: true,
  },
+  {
+    year: 2024,
+    title: "Transfusion: Predict the Next Token and Diffuse Images with One Multi-Modal Model",
+    url: "https://arxiv.org/abs/2408.11039",
+    topics: ["多模态", "Transformer"],
+    contribution: "在同一 Transformer 上对文字做 NTP、对连续图像表示做 diffusion。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution",
+    url: "https://arxiv.org/abs/2409.12191",
+    topics: ["多模态", "长上下文"],
+    contribution: "引入动态视觉 Token、M-RoPE 与统一图像 / 视频输入。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation",
+    url: "https://arxiv.org/abs/2410.13848",
+    topics: ["多模态"],
+    contribution: "解耦理解与生成的视觉编码路径，同时共享一个自回归主干。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "JanusFlow: Harmonizing Autoregression and Rectified Flow for Unified Multimodal Understanding and Generation",
+    url: "https://arxiv.org/abs/2411.07975",
+    topics: ["多模态"],
+    contribution: "把语言自回归与图像 rectified flow 放进同一框架，并对齐解耦视觉表示。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Expanding Performance Boundaries of Open-Source Multimodal Models with Model, Data, and Test-Time Scaling",
+    url: "https://arxiv.org/abs/2412.05271",
+    topics: ["多模态", "Scaling", "推理"],
+    contribution: "InternVL 2.5 从模型、数据与测试时配置三个轴研究多模态扩展。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "DeepSeek-VL2: Mixture-of-Experts Vision-Language Models for Advanced Multimodal Understanding",
+    url: "https://arxiv.org/abs/2412.10302",
+    topics: ["多模态", "MoE", "长上下文"],
+    contribution: "用动态 tile、2×2 pixel shuffle、MLP adaptor、DeepSeekMoE 与 MLA 处理高分辨率输入。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "Janus-Pro: Unified Multimodal Understanding and Generation with Data and Model Scaling",
+    url: "https://arxiv.org/abs/2501.17811",
+    topics: ["多模态", "Scaling"],
+    contribution: "延续 Janus 解耦视觉路径，改进训练策略、数据规模与模型规模。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "Qwen2.5-VL Technical Report",
+    url: "https://arxiv.org/abs/2502.13923",
+    topics: ["多模态", "Agent"],
+    contribution: "从头训练动态分辨率 ViT，引入绝对时间编码并强化文档和视觉 Agent。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "Vision-R1: Incentivizing Reasoning Capability in Multimodal Large Language Models",
+    url: "https://arxiv.org/abs/2503.06749",
+    topics: ["多模态", "推理", "后训练"],
+    contribution: "探索用大规模强化学习激活多模态长思维与视觉推理。",
+    verified: true,
+  },
  {
    year: 2025,
    title: "Kimi-VL Technical Report",
@@ -2481,6 +2782,40 @@ export const papers: Paper[] = [
    spotlight: "Kimi",
    verified: true,
  },
+  {
+    year: 2025,
+    title: "VTool-R1: VLMs Learn to Think with Images via Reinforcement Learning on Multimodal Tool Use",
+    url: "https://arxiv.org/abs/2505.19255",
+    topics: ["多模态", "推理", "Agent", "后训练"],
+    contribution: "用可验证奖励训练模型自主选择视觉工具并从新图像观察继续推理。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "DeepSeek-OCR: Contexts Optical Compression",
+    url: "https://arxiv.org/abs/2510.18234",
+    topics: ["多模态", "长上下文", "数据"],
+    contribution: "用 DeepEncoder 将二维页面压成少量视觉 Token，研究光学长上下文压缩。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2026,
+    title: "DeepSeek-OCR 2: Visual Causal Flow",
+    url: "https://arxiv.org/abs/2601.20552",
+    topics: ["多模态", "长上下文"],
+    contribution: "DeepEncoder V2 根据图像语义动态重排视觉 Token，再交给语言解码器。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2026,
+    title: "ToolsRL: Reward Is All Tool Learning Needs",
+    url: "https://arxiv.org/abs/2604.19945",
+    topics: ["多模态", "推理", "Agent", "后训练"],
+    contribution: "以强化学习研究视觉工具选择、调用与多步观察闭环。",
+    verified: true,
+  },
  {
    year: 2020,
    title: "Measuring Massive Multitask Language Understanding",