Refresh papers and define Agent evaluation

tools/collection/README.md

@@ -63,6 +63,23 @@ python3 tools/collection/collect_arxiv.py \
 - 自动分层：生成 `queued` 条目，后续人工精读后再升级状态。
 - API 退避：遇到 arXiv 429 或临时网络错误时按 `--retries` 和 `--retry-sleep` 重试。
 
+当 `export.arxiv.org` API 不可用时，可以使用 arXiv 官方搜索页面后端：
+
+```bash
+python3 tools/collection/collect_arxiv.py \
+  --backend search-html \
+  --from-date 2026-07-09 \
+  --to-date 2026-07-27 \
+  --per-query 300 \
+  --page-size 100 \
+  --request-timeout 120 \
+  --dry-run
+```
+
+`--dry-run` 不写 `papers/items/`，但会保存候选和选择 manifest。之后使用
+`promote_arxiv_manifest.py` 按本地阈值入库。API 和 HTML 后端使用相同的分类、去重和
+manifest schema。
+
 ## Method
 
 1. 搜索或导入资料。

tools/collection/collect_arxiv.py

@@ -4,20 +4,24 @@
 from __future__ import annotations
 
 import argparse
+import http.client
 import json
 import re
+import sys
 import time
 import urllib.parse
 import urllib.request
 import urllib.error
 import xml.etree.ElementTree as ET
 from collections import defaultdict
-from datetime import date
+from datetime import date, datetime
+from html.parser import HTMLParser
 from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[2]
 API_URL = "https://export.arxiv.org/api/query"
+SEARCH_URL = "https://arxiv.org/search/"
 NS = {
     "atom": "http://www.w3.org/2005/Atom",
     "arxiv": "http://arxiv.org/schemas/atom",
@@ -42,6 +46,24 @@ QUERIES = [
     ("autonomous-agent-llm", 'all:"autonomous agent" AND (all:"LLM" OR all:"large language model")'),
 ]
 
+SEARCH_QUERIES = {
+    "llm-agent": '"LLM agent" OR "LLM agents"',
+    "language-agent": '"language agent" OR "language agents"',
+    "ai-agent": '"AI agent" OR "AI agents"',
+    "agentic-ai": '"agentic AI" OR "agentic workflow"',
+    "agent-evaluation": '"agent evaluation" OR "agent benchmark" OR "agentic benchmark"',
+    "agent-memory": '"agent memory" OR "memory agent"',
+    "tool-use": '"tool use" agent',
+    "function-calling": '"function calling" agent',
+    "coding-agent": '"coding agent" OR "software engineering agent" OR SWE-bench',
+    "web-gui-agent": '"web agent" OR "browser agent" OR "GUI agent" OR "computer use"',
+    "multi-agent-llm": '"multi-agent" LLM',
+    "agent-safety": '"agent safety" OR "agent security"',
+    "rag-agent": "RAG agent",
+    "planning-agent": 'planning "LLM agent"',
+    "autonomous-agent-llm": '"autonomous agent" LLM',
+}
+
 
 TOPIC_RULES = [
     ("agent-evaluation", ("evaluation", "benchmark", "eval", "metric", "leaderboard", "assessment")),
@@ -118,6 +140,7 @@ def fetch_query(
     max_results: int,
     retries: int,
     retry_sleep: float,
+    request_timeout: float,
 ) -> str:
     params = {
         "search_query": date_window_query(raw_query, from_date, to_date),
@@ -130,19 +153,219 @@ def fetch_query(
     request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.1"})
     for attempt in range(retries + 1):
         try:
-            with urllib.request.urlopen(request, timeout=30) as response:
+            with urllib.request.urlopen(request, timeout=request_timeout) as response:
                 return response.read().decode("utf-8", "replace")
         except urllib.error.HTTPError as exc:
-            if exc.code != 429 or attempt >= retries:
+            if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
                 raise
+            print(
+                f"arXiv returned HTTP {exc.code}; retrying request "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
             time.sleep(retry_sleep * (attempt + 1))
-        except urllib.error.URLError:
+        except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
             if attempt >= retries:
                 raise
+            print(
+                f"arXiv request failed with {type(exc).__name__}; retrying "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
             time.sleep(retry_sleep * (attempt + 1))
     raise RuntimeError("unreachable fetch retry state")
 
 
+class ArxivSearchParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.records: list[dict] = []
+        self.current: dict | None = None
+        self.capture = ""
+        self.capture_tag = ""
+        self.capture_depth = 0
+        self.buffer: list[str] = []
+
+    @staticmethod
+    def classes(attrs: list[tuple[str, str | None]]) -> set[str]:
+        values = dict(attrs).get("class") or ""
+        return set(values.split())
+
+    def start_capture(self, field: str, tag: str) -> None:
+        self.capture = field
+        self.capture_tag = tag
+        self.capture_depth = 1
+        self.buffer = []
+
+    def finish_capture(self) -> None:
+        if self.current is None:
+            return
+        value = normalize_space("".join(self.buffer))
+        if self.capture == "authors":
+            value = re.sub(r"^Authors:\s*", "", value)
+            self.current["authors"] = [part.strip() for part in value.split(",") if part.strip()]
+        elif self.capture == "summary":
+            self.current["summary"] = re.sub(r"(?:△|▽)?\s*Less\s*$", "", value).strip()
+        elif self.capture == "category":
+            if value:
+                self.current["categories"].append(value)
+        else:
+            self.current[self.capture] = value
+        self.capture = ""
+        self.capture_tag = ""
+        self.capture_depth = 0
+        self.buffer = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        classes = self.classes(attrs)
+        if tag == "li" and "arxiv-result" in classes:
+            self.current = {
+                "id": "",
+                "url": "",
+                "title": "",
+                "summary": "",
+                "submitted": "",
+                "authors": [],
+                "categories": [],
+            }
+            return
+        if self.current is None:
+            return
+        if self.capture:
+            self.capture_depth += 1
+        elif tag == "p" and "title" in classes:
+            self.start_capture("title", tag)
+        elif tag == "p" and "authors" in classes:
+            self.start_capture("authors", tag)
+        elif tag == "span" and "abstract-full" in classes:
+            self.start_capture("summary", tag)
+        elif tag == "span" and "tag" in classes and "is-link" in classes:
+            self.start_capture("category", tag)
+        elif tag == "p" and "is-size-7" in classes:
+            self.start_capture("submitted", tag)
+
+        href = dict(attrs).get("href") or ""
+        match = re.fullmatch(r"https://arxiv\.org/abs/([0-9]+\.[0-9]+)(?:v[0-9]+)?", href)
+        if tag == "a" and match and not self.current["id"]:
+            self.current["id"] = match.group(1)
+            self.current["url"] = f"https://arxiv.org/abs/{match.group(1)}"
+
+    def handle_endtag(self, tag: str) -> None:
+        if self.current is None:
+            return
+        if self.capture:
+            self.capture_depth -= 1
+            if self.capture_depth == 0 and tag == self.capture_tag:
+                self.finish_capture()
+        if tag == "li":
+            record = self.current
+            self.current = None
+            if record["id"] and record["title"]:
+                submitted_match = re.search(
+                    r"Submitted\s+([0-9]{1,2}\s+[A-Za-z]+,\s+[0-9]{4})",
+                    record.pop("submitted", ""),
+                )
+                if submitted_match:
+                    published = datetime.strptime(submitted_match.group(1), "%d %B, %Y").date().isoformat()
+                    record["published"] = published
+                    record["updated"] = published
+                    self.records.append(record)
+
+    def handle_data(self, data: str) -> None:
+        if self.capture:
+            self.buffer.append(data)
+
+
+def fetch_search_page(
+    query: str,
+    start: int,
+    max_results: int,
+    retries: int,
+    retry_sleep: float,
+    request_timeout: float,
+) -> str:
+    params = {
+        "query": query,
+        "searchtype": "all",
+        "abstracts": "show",
+        "order": "-announced_date_first",
+        "size": min(max_results, 200),
+        "start": start,
+    }
+    url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"
+    request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.2"})
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(request, timeout=request_timeout) as response:
+                return response.read().decode("utf-8", "replace")
+        except urllib.error.HTTPError as exc:
+            if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
+                raise
+            print(
+                f"arXiv search returned HTTP {exc.code}; retrying request "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
+        except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
+            if attempt >= retries:
+                raise
+            print(
+                f"arXiv search failed with {type(exc).__name__}; retrying "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
+        time.sleep(retry_sleep * (attempt + 1))
+    raise RuntimeError("unreachable search retry state")
+
+
+def fetch_search_query(
+    query: str,
+    from_date: str,
+    to_date: str,
+    per_query: int,
+    page_size: int,
+    sleep_seconds: float,
+    retries: int,
+    retry_sleep: float,
+    request_timeout: float,
+) -> list[dict]:
+    records: list[dict] = []
+    start = 0
+    while start < per_query:
+        requested = min(page_size, per_query - start, 200)
+        html_text = fetch_search_page(
+            query,
+            start,
+            requested,
+            retries,
+            retry_sleep,
+            request_timeout,
+        )
+        parser = ArxivSearchParser()
+        parser.feed(html_text)
+        page_records = parser.records
+        if not page_records:
+            break
+        records.extend(record for record in page_records if from_date <= record["published"] <= to_date)
+        print(
+            f"searched arXiv HTML: {start + len(page_records)} records, "
+            f"{len(records)} inside window",
+            file=sys.stderr,
+            flush=True,
+        )
+        start += len(page_records)
+        if len(page_records) < requested:
+            break
+        # Announced-date ordering can mix old cross-listed papers into a recent page,
+        # so the original submission date is not a safe early-stop signal.
+        time.sleep(sleep_seconds)
+    return records
+
+
 def manifest_record(record: dict, topics: list[str], score: int, relevance: str, matched_queries: set[str]) -> dict:
     sorted_queries = sorted(matched_queries)
     return {
@@ -338,6 +561,8 @@ def main() -> int:
     parser.add_argument("--sleep", type=float, default=1.0)
     parser.add_argument("--retries", type=int, default=4)
     parser.add_argument("--retry-sleep", type=float, default=10.0)
+    parser.add_argument("--request-timeout", type=float, default=60.0)
+    parser.add_argument("--backend", choices=("api", "search-html"), default="api")
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
 
@@ -345,8 +570,26 @@ def main() -> int:
     matches: dict[str, set[str]] = defaultdict(set)
 
     for label, raw_query in QUERIES:
+        if args.backend == "search-html":
+            print(f"collecting {label} from arXiv search", file=sys.stderr, flush=True)
+            records = fetch_search_query(
+                SEARCH_QUERIES[label],
+                args.from_date,
+                args.to_date,
+                args.per_query,
+                args.page_size,
+                args.sleep,
+                args.retries,
+                args.retry_sleep,
+                args.request_timeout,
+            )
+            for record in records:
+                by_id.setdefault(record["id"], record)
+                matches[record["id"]].add(label)
+            continue
         fetched = 0
         start = 0
+        print(f"collecting {label}", file=sys.stderr, flush=True)
         while fetched < args.per_query:
             page_size = min(args.page_size, args.per_query - fetched)
             xml_text = fetch_query(
@@ -357,6 +600,7 @@ def main() -> int:
                 page_size,
                 args.retries,
                 args.retry_sleep,
+                args.request_timeout,
             )
             records = parse_feed(xml_text)
             if not records:
@@ -366,6 +610,11 @@ def main() -> int:
                 matches[record["id"]].add(label)
             fetched += len(records)
             start += len(records)
+            print(
+                f"collected {label}: {fetched} records",
+                file=sys.stderr,
+                flush=True,
+            )
             if len(records) < page_size:
                 break
             time.sleep(args.sleep)

tools/collection/test_collect_arxiv.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import unittest
+
+from tools.collection.collect_arxiv import ArxivSearchParser, classify
+
+
+SEARCH_RESULT = """
+<ol>
+  <li class="arxiv-result">
+    <p class="list-title is-inline-block">
+      <a href="https://arxiv.org/abs/2607.12345v2">arXiv:2607.12345</a>
+    </p>
+    <p class="title is-5 mathjax">
+      A Trace-Grounded Benchmark for Tool-Using Agents
+    </p>
+    <p class="authors">
+      <span>Authors:</span>
+      <a>Alice Example</a>, <a>Bob Example</a>
+    </p>
+    <div class="tags">
+      <span class="tag is-small is-link tooltip">cs.AI</span>
+      <span class="tag is-small is-link tooltip">cs.LG</span>
+    </div>
+    <span class="abstract-full has-text-grey-dark mathjax">
+      We evaluate an LLM agent with tools and deterministic verification.
+      <a>Less</a>
+    </span>
+    <p class="is-size-7">Submitted 24 July, 2026; originally announced July 2026.</p>
+  </li>
+</ol>
+"""
+
+
+class ArxivSearchParserTest(unittest.TestCase):
+    def test_parses_search_result_into_api_compatible_record(self) -> None:
+        parser = ArxivSearchParser()
+        parser.feed(SEARCH_RESULT)
+
+        self.assertEqual(len(parser.records), 1)
+        record = parser.records[0]
+        self.assertEqual(record["id"], "2607.12345")
+        self.assertEqual(record["url"], "https://arxiv.org/abs/2607.12345")
+        self.assertEqual(record["published"], "2026-07-24")
+        self.assertEqual(record["updated"], "2026-07-24")
+        self.assertEqual(record["authors"], ["Alice Example", "Bob Example"])
+        self.assertEqual(record["categories"], ["cs.AI", "cs.LG"])
+        self.assertNotIn("Less", record["summary"])
+
+    def test_parsed_record_uses_existing_classifier(self) -> None:
+        parser = ArxivSearchParser()
+        parser.feed(SEARCH_RESULT)
+        topics, score, relevance = classify(parser.records[0], {"agent-evaluation", "tool-use"})
+
+        self.assertIn("agent-evaluation", topics)
+        self.assertIn("tool-use", topics)
+        self.assertGreaterEqual(score, 7)
+        self.assertIn(relevance, {"medium", "high"})
+
+
+if __name__ == "__main__":
+    unittest.main()