Refresh papers and define Agent evaluation

2026-07-27 16:28:23 +08:00
parent 8e4ff3779b
commit df475e8d90
180 changed files with 31313 additions and 160 deletions
@@ -4,20 +4,24 @@
 from __future__ import annotations

 import argparse
+import http.client
 import json
 import re
+import sys
 import time
 import urllib.parse
 import urllib.request
 import urllib.error
 import xml.etree.ElementTree as ET
 from collections import defaultdict
-from datetime import date
+from datetime import date, datetime
+from html.parser import HTMLParser
 from pathlib import Path


 ROOT = Path(__file__).resolve().parents[2]
 API_URL = "https://export.arxiv.org/api/query"
+SEARCH_URL = "https://arxiv.org/search/"
 NS = {
    "atom": "http://www.w3.org/2005/Atom",
    "arxiv": "http://arxiv.org/schemas/atom",
@@ -42,6 +46,24 @@ QUERIES = [
    ("autonomous-agent-llm", 'all:"autonomous agent" AND (all:"LLM" OR all:"large language model")'),
 ]

+SEARCH_QUERIES = {
+    "llm-agent": '"LLM agent" OR "LLM agents"',
+    "language-agent": '"language agent" OR "language agents"',
+    "ai-agent": '"AI agent" OR "AI agents"',
+    "agentic-ai": '"agentic AI" OR "agentic workflow"',
+    "agent-evaluation": '"agent evaluation" OR "agent benchmark" OR "agentic benchmark"',
+    "agent-memory": '"agent memory" OR "memory agent"',
+    "tool-use": '"tool use" agent',
+    "function-calling": '"function calling" agent',
+    "coding-agent": '"coding agent" OR "software engineering agent" OR SWE-bench',
+    "web-gui-agent": '"web agent" OR "browser agent" OR "GUI agent" OR "computer use"',
+    "multi-agent-llm": '"multi-agent" LLM',
+    "agent-safety": '"agent safety" OR "agent security"',
+    "rag-agent": "RAG agent",
+    "planning-agent": 'planning "LLM agent"',
+    "autonomous-agent-llm": '"autonomous agent" LLM',
+}
+

 TOPIC_RULES = [
    ("agent-evaluation", ("evaluation", "benchmark", "eval", "metric", "leaderboard", "assessment")),
@@ -118,6 +140,7 @@ def fetch_query(
    max_results: int,
    retries: int,
    retry_sleep: float,
+    request_timeout: float,
 ) -> str:
    params = {
        "search_query": date_window_query(raw_query, from_date, to_date),
@@ -130,19 +153,219 @@ def fetch_query(
    request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.1"})
    for attempt in range(retries + 1):
        try:
-            with urllib.request.urlopen(request, timeout=30) as response:
+            with urllib.request.urlopen(request, timeout=request_timeout) as response:
                return response.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as exc:
-            if exc.code != 429 or attempt >= retries:
+            if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
                raise
+            print(
+                f"arXiv returned HTTP {exc.code}; retrying request "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
            time.sleep(retry_sleep * (attempt + 1))
-        except urllib.error.URLError:
+        except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
            if attempt >= retries:
                raise
+            print(
+                f"arXiv request failed with {type(exc).__name__}; retrying "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
            time.sleep(retry_sleep * (attempt + 1))
    raise RuntimeError("unreachable fetch retry state")


+class ArxivSearchParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.records: list[dict] = []
+        self.current: dict | None = None
+        self.capture = ""
+        self.capture_tag = ""
+        self.capture_depth = 0
+        self.buffer: list[str] = []
+
+    @staticmethod
+    def classes(attrs: list[tuple[str, str | None]]) -> set[str]:
+        values = dict(attrs).get("class") or ""
+        return set(values.split())
+
+    def start_capture(self, field: str, tag: str) -> None:
+        self.capture = field
+        self.capture_tag = tag
+        self.capture_depth = 1
+        self.buffer = []
+
+    def finish_capture(self) -> None:
+        if self.current is None:
+            return
+        value = normalize_space("".join(self.buffer))
+        if self.capture == "authors":
+            value = re.sub(r"^Authors:\s*", "", value)
+            self.current["authors"] = [part.strip() for part in value.split(",") if part.strip()]
+        elif self.capture == "summary":
+            self.current["summary"] = re.sub(r"(?:△|▽)?\s*Less\s*$", "", value).strip()
+        elif self.capture == "category":
+            if value:
+                self.current["categories"].append(value)
+        else:
+            self.current[self.capture] = value
+        self.capture = ""
+        self.capture_tag = ""
+        self.capture_depth = 0
+        self.buffer = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        classes = self.classes(attrs)
+        if tag == "li" and "arxiv-result" in classes:
+            self.current = {
+                "id": "",
+                "url": "",
+                "title": "",
+                "summary": "",
+                "submitted": "",
+                "authors": [],
+                "categories": [],
+            }
+            return
+        if self.current is None:
+            return
+        if self.capture:
+            self.capture_depth += 1
+        elif tag == "p" and "title" in classes:
+            self.start_capture("title", tag)
+        elif tag == "p" and "authors" in classes:
+            self.start_capture("authors", tag)
+        elif tag == "span" and "abstract-full" in classes:
+            self.start_capture("summary", tag)
+        elif tag == "span" and "tag" in classes and "is-link" in classes:
+            self.start_capture("category", tag)
+        elif tag == "p" and "is-size-7" in classes:
+            self.start_capture("submitted", tag)
+
+        href = dict(attrs).get("href") or ""
+        match = re.fullmatch(r"https://arxiv\.org/abs/([0-9]+\.[0-9]+)(?:v[0-9]+)?", href)
+        if tag == "a" and match and not self.current["id"]:
+            self.current["id"] = match.group(1)
+            self.current["url"] = f"https://arxiv.org/abs/{match.group(1)}"
+
+    def handle_endtag(self, tag: str) -> None:
+        if self.current is None:
+            return
+        if self.capture:
+            self.capture_depth -= 1
+            if self.capture_depth == 0 and tag == self.capture_tag:
+                self.finish_capture()
+        if tag == "li":
+            record = self.current
+            self.current = None
+            if record["id"] and record["title"]:
+                submitted_match = re.search(
+                    r"Submitted\s+([0-9]{1,2}\s+[A-Za-z]+,\s+[0-9]{4})",
+                    record.pop("submitted", ""),
+                )
+                if submitted_match:
+                    published = datetime.strptime(submitted_match.group(1), "%d %B, %Y").date().isoformat()
+                    record["published"] = published
+                    record["updated"] = published
+                    self.records.append(record)
+
+    def handle_data(self, data: str) -> None:
+        if self.capture:
+            self.buffer.append(data)
+
+
+def fetch_search_page(
+    query: str,
+    start: int,
+    max_results: int,
+    retries: int,
+    retry_sleep: float,
+    request_timeout: float,
+) -> str:
+    params = {
+        "query": query,
+        "searchtype": "all",
+        "abstracts": "show",
+        "order": "-announced_date_first",
+        "size": min(max_results, 200),
+        "start": start,
+    }
+    url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"
+    request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.2"})
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(request, timeout=request_timeout) as response:
+                return response.read().decode("utf-8", "replace")
+        except urllib.error.HTTPError as exc:
+            if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
+                raise
+            print(
+                f"arXiv search returned HTTP {exc.code}; retrying request "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
+        except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
+            if attempt >= retries:
+                raise
+            print(
+                f"arXiv search failed with {type(exc).__name__}; retrying "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
+        time.sleep(retry_sleep * (attempt + 1))
+    raise RuntimeError("unreachable search retry state")
+
+
+def fetch_search_query(
+    query: str,
+    from_date: str,
+    to_date: str,
+    per_query: int,
+    page_size: int,
+    sleep_seconds: float,
+    retries: int,
+    retry_sleep: float,
+    request_timeout: float,
+) -> list[dict]:
+    records: list[dict] = []
+    start = 0
+    while start < per_query:
+        requested = min(page_size, per_query - start, 200)
+        html_text = fetch_search_page(
+            query,
+            start,
+            requested,
+            retries,
+            retry_sleep,
+            request_timeout,
+        )
+        parser = ArxivSearchParser()
+        parser.feed(html_text)
+        page_records = parser.records
+        if not page_records:
+            break
+        records.extend(record for record in page_records if from_date <= record["published"] <= to_date)
+        print(
+            f"searched arXiv HTML: {start + len(page_records)} records, "
+            f"{len(records)} inside window",
+            file=sys.stderr,
+            flush=True,
+        )
+        start += len(page_records)
+        if len(page_records) < requested:
+            break
+        # Announced-date ordering can mix old cross-listed papers into a recent page,
+        # so the original submission date is not a safe early-stop signal.
+        time.sleep(sleep_seconds)
+    return records
+
+
 def manifest_record(record: dict, topics: list[str], score: int, relevance: str, matched_queries: set[str]) -> dict:
    sorted_queries = sorted(matched_queries)
    return {
@@ -338,6 +561,8 @@ def main() -> int:
    parser.add_argument("--sleep", type=float, default=1.0)
    parser.add_argument("--retries", type=int, default=4)
    parser.add_argument("--retry-sleep", type=float, default=10.0)
+    parser.add_argument("--request-timeout", type=float, default=60.0)
+    parser.add_argument("--backend", choices=("api", "search-html"), default="api")
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()

@@ -345,8 +570,26 @@ def main() -> int:
    matches: dict[str, set[str]] = defaultdict(set)

    for label, raw_query in QUERIES:
+        if args.backend == "search-html":
+            print(f"collecting {label} from arXiv search", file=sys.stderr, flush=True)
+            records = fetch_search_query(
+                SEARCH_QUERIES[label],
+                args.from_date,
+                args.to_date,
+                args.per_query,
+                args.page_size,
+                args.sleep,
+                args.retries,
+                args.retry_sleep,
+                args.request_timeout,
+            )
+            for record in records:
+                by_id.setdefault(record["id"], record)
+                matches[record["id"]].add(label)
+            continue
        fetched = 0
        start = 0
+        print(f"collecting {label}", file=sys.stderr, flush=True)
        while fetched < args.per_query:
            page_size = min(args.page_size, args.per_query - fetched)
            xml_text = fetch_query(
@@ -357,6 +600,7 @@ def main() -> int:
                page_size,
                args.retries,
                args.retry_sleep,
+                args.request_timeout,
            )
            records = parse_feed(xml_text)
            if not records:
@@ -366,6 +610,11 @@ def main() -> int:
                matches[record["id"]].add(label)
            fetched += len(records)
            start += len(records)
+            print(
+                f"collected {label}: {fetched} records",
+                file=sys.stderr,
+                flush=True,
+            )
            if len(records) < page_size:
                break
            time.sleep(args.sleep)