wuyang
675 lines
24 KiB
Python
675 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Collect recent arXiv papers related to LLM/AI agents."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import urllib.error
|
|
import xml.etree.ElementTree as ET
|
|
from collections import defaultdict
|
|
from datetime import date, datetime
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
API_URL = "https://export.arxiv.org/api/query"
|
|
SEARCH_URL = "https://arxiv.org/search/"
|
|
NS = {
|
|
"atom": "http://www.w3.org/2005/Atom",
|
|
"arxiv": "http://arxiv.org/schemas/atom",
|
|
}
|
|
|
|
|
|
QUERIES = [
|
|
("llm-agent", 'all:"LLM agent" OR all:"LLM agents"'),
|
|
("language-agent", 'all:"language agent" OR all:"language agents"'),
|
|
("ai-agent", 'all:"AI agent" OR all:"AI agents"'),
|
|
("agentic-ai", 'all:"agentic AI" OR all:"agentic workflow"'),
|
|
("agent-evaluation", 'all:"agent evaluation" OR all:"agent benchmark" OR all:"agentic benchmark"'),
|
|
("agent-memory", 'all:"agent memory" OR all:"memory agent" OR all:"memory agents"'),
|
|
("tool-use", 'all:"tool use" AND all:"agent"'),
|
|
("function-calling", 'all:"function calling" AND all:"agent"'),
|
|
("coding-agent", 'all:"coding agent" OR all:"software engineering agent" OR all:"SWE-bench"'),
|
|
("web-gui-agent", 'all:"web agent" OR all:"browser agent" OR all:"GUI agent" OR all:"computer use"'),
|
|
("multi-agent-llm", 'all:"multi-agent" AND (all:"LLM" OR all:"large language model")'),
|
|
("agent-safety", 'all:"agent safety" OR all:"AI agent safety" OR all:"agent security"'),
|
|
("rag-agent", 'all:"RAG" AND all:"agent"'),
|
|
("planning-agent", 'all:"planning" AND all:"LLM agent"'),
|
|
("autonomous-agent-llm", 'all:"autonomous agent" AND (all:"LLM" OR all:"large language model")'),
|
|
]
|
|
|
|
SEARCH_QUERIES = {
|
|
"llm-agent": '"LLM agent" OR "LLM agents"',
|
|
"language-agent": '"language agent" OR "language agents"',
|
|
"ai-agent": '"AI agent" OR "AI agents"',
|
|
"agentic-ai": '"agentic AI" OR "agentic workflow"',
|
|
"agent-evaluation": '"agent evaluation" OR "agent benchmark" OR "agentic benchmark"',
|
|
"agent-memory": '"agent memory" OR "memory agent"',
|
|
"tool-use": '"tool use" agent',
|
|
"function-calling": '"function calling" agent',
|
|
"coding-agent": '"coding agent" OR "software engineering agent" OR SWE-bench',
|
|
"web-gui-agent": '"web agent" OR "browser agent" OR "GUI agent" OR "computer use"',
|
|
"multi-agent-llm": '"multi-agent" LLM',
|
|
"agent-safety": '"agent safety" OR "agent security"',
|
|
"rag-agent": "RAG agent",
|
|
"planning-agent": 'planning "LLM agent"',
|
|
"autonomous-agent-llm": '"autonomous agent" LLM',
|
|
}
|
|
|
|
|
|
TOPIC_RULES = [
|
|
("agent-evaluation", ("evaluation", "benchmark", "eval", "metric", "leaderboard", "assessment")),
|
|
("memory", ("memory", "remember", "stateful", "long-term", "episodic")),
|
|
("tool-use", ("tool", "function calling", "api", "mcp", "action")),
|
|
("coding-agent", ("coding", "software engineering", "swe-bench", "repository", "program repair", "code agent")),
|
|
("computer-use", ("computer use", "gui", "browser", "web agent", "mobile", "desktop")),
|
|
("multi-agent", ("multi-agent", "multiagent", "agent society", "collaboration", "debate")),
|
|
("agent-safety", ("safety", "security", "risk", "prompt injection", "alignment", "guardrail")),
|
|
("rag", ("retrieval", "rag", "knowledge base", "grounding")),
|
|
("planning", ("planning", "planner", "plan", "long-horizon", "task decomposition")),
|
|
("reasoning", ("reasoning", "reflection", "self-improvement", "verifier")),
|
|
("workflow-agent", ("workflow", "enterprise", "office", "productivity", "automation")),
|
|
("embodied-agent", ("embodied", "robot", "robotic", "navigation")),
|
|
("world-model", ("world model", "simulation", "environment model")),
|
|
]
|
|
|
|
|
|
HIGH_SIGNAL_TERMS = (
|
|
"llm agent",
|
|
"language agent",
|
|
"ai agent",
|
|
"agentic",
|
|
"tool use",
|
|
"memory",
|
|
"benchmark",
|
|
"evaluation",
|
|
"coding agent",
|
|
"swe-bench",
|
|
"computer use",
|
|
"gui agent",
|
|
"multi-agent",
|
|
"agent safety",
|
|
)
|
|
|
|
|
|
def normalize_space(value: str) -> str:
|
|
return re.sub(r"\s+", " ", value).strip()
|
|
|
|
|
|
def slugify(text: str) -> str:
|
|
text = text.lower()
|
|
text = re.sub(r"[^a-z0-9]+", "-", text)
|
|
return re.sub(r"-+", "-", text).strip("-")[:80] or "untitled"
|
|
|
|
|
|
def yaml_value(value: str | int | None) -> str:
|
|
if value is None:
|
|
return ""
|
|
text = str(value).replace("\n", " ").strip()
|
|
if not text:
|
|
return ""
|
|
if any(char in text for char in [":", "#", "[", "]", "{", "}", "\"", "'"]):
|
|
return json.dumps(text, ensure_ascii=False)
|
|
return text
|
|
|
|
|
|
def arxiv_id(entry_id: str) -> str:
|
|
raw = entry_id.rsplit("/", 1)[-1]
|
|
return raw.split("v", 1)[0]
|
|
|
|
|
|
def date_window_query(raw_query: str, from_date: str, to_date: str) -> str:
|
|
start = from_date.replace("-", "") + "0000"
|
|
end = to_date.replace("-", "") + "2359"
|
|
return f"({raw_query}) AND submittedDate:[{start} TO {end}]"
|
|
|
|
|
|
def fetch_query(
|
|
raw_query: str,
|
|
from_date: str,
|
|
to_date: str,
|
|
start: int,
|
|
max_results: int,
|
|
retries: int,
|
|
retry_sleep: float,
|
|
request_timeout: float,
|
|
) -> str:
|
|
params = {
|
|
"search_query": date_window_query(raw_query, from_date, to_date),
|
|
"start": start,
|
|
"max_results": max_results,
|
|
"sortBy": "submittedDate",
|
|
"sortOrder": "descending",
|
|
}
|
|
url = f"{API_URL}?{urllib.parse.urlencode(params)}"
|
|
request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.1"})
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=request_timeout) as response:
|
|
return response.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
|
|
raise
|
|
print(
|
|
f"arXiv returned HTTP {exc.code}; retrying request "
|
|
f"{attempt + 1}/{retries} after backoff",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
time.sleep(retry_sleep * (attempt + 1))
|
|
except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
|
|
if attempt >= retries:
|
|
raise
|
|
print(
|
|
f"arXiv request failed with {type(exc).__name__}; retrying "
|
|
f"{attempt + 1}/{retries} after backoff",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
time.sleep(retry_sleep * (attempt + 1))
|
|
raise RuntimeError("unreachable fetch retry state")
|
|
|
|
|
|
class ArxivSearchParser(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.records: list[dict] = []
|
|
self.current: dict | None = None
|
|
self.capture = ""
|
|
self.capture_tag = ""
|
|
self.capture_depth = 0
|
|
self.buffer: list[str] = []
|
|
|
|
@staticmethod
|
|
def classes(attrs: list[tuple[str, str | None]]) -> set[str]:
|
|
values = dict(attrs).get("class") or ""
|
|
return set(values.split())
|
|
|
|
def start_capture(self, field: str, tag: str) -> None:
|
|
self.capture = field
|
|
self.capture_tag = tag
|
|
self.capture_depth = 1
|
|
self.buffer = []
|
|
|
|
def finish_capture(self) -> None:
|
|
if self.current is None:
|
|
return
|
|
value = normalize_space("".join(self.buffer))
|
|
if self.capture == "authors":
|
|
value = re.sub(r"^Authors:\s*", "", value)
|
|
self.current["authors"] = [part.strip() for part in value.split(",") if part.strip()]
|
|
elif self.capture == "summary":
|
|
self.current["summary"] = re.sub(r"(?:△|▽)?\s*Less\s*$", "", value).strip()
|
|
elif self.capture == "category":
|
|
if value:
|
|
self.current["categories"].append(value)
|
|
else:
|
|
self.current[self.capture] = value
|
|
self.capture = ""
|
|
self.capture_tag = ""
|
|
self.capture_depth = 0
|
|
self.buffer = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
classes = self.classes(attrs)
|
|
if tag == "li" and "arxiv-result" in classes:
|
|
self.current = {
|
|
"id": "",
|
|
"url": "",
|
|
"title": "",
|
|
"summary": "",
|
|
"submitted": "",
|
|
"authors": [],
|
|
"categories": [],
|
|
}
|
|
return
|
|
if self.current is None:
|
|
return
|
|
if self.capture:
|
|
self.capture_depth += 1
|
|
elif tag == "p" and "title" in classes:
|
|
self.start_capture("title", tag)
|
|
elif tag == "p" and "authors" in classes:
|
|
self.start_capture("authors", tag)
|
|
elif tag == "span" and "abstract-full" in classes:
|
|
self.start_capture("summary", tag)
|
|
elif tag == "span" and "tag" in classes and "is-link" in classes:
|
|
self.start_capture("category", tag)
|
|
elif tag == "p" and "is-size-7" in classes:
|
|
self.start_capture("submitted", tag)
|
|
|
|
href = dict(attrs).get("href") or ""
|
|
match = re.fullmatch(r"https://arxiv\.org/abs/([0-9]+\.[0-9]+)(?:v[0-9]+)?", href)
|
|
if tag == "a" and match and not self.current["id"]:
|
|
self.current["id"] = match.group(1)
|
|
self.current["url"] = f"https://arxiv.org/abs/{match.group(1)}"
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if self.current is None:
|
|
return
|
|
if self.capture:
|
|
self.capture_depth -= 1
|
|
if self.capture_depth == 0 and tag == self.capture_tag:
|
|
self.finish_capture()
|
|
if tag == "li":
|
|
record = self.current
|
|
self.current = None
|
|
if record["id"] and record["title"]:
|
|
submitted_match = re.search(
|
|
r"Submitted\s+([0-9]{1,2}\s+[A-Za-z]+,\s+[0-9]{4})",
|
|
record.pop("submitted", ""),
|
|
)
|
|
if submitted_match:
|
|
published = datetime.strptime(submitted_match.group(1), "%d %B, %Y").date().isoformat()
|
|
record["published"] = published
|
|
record["updated"] = published
|
|
self.records.append(record)
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self.capture:
|
|
self.buffer.append(data)
|
|
|
|
|
|
def fetch_search_page(
|
|
query: str,
|
|
start: int,
|
|
max_results: int,
|
|
retries: int,
|
|
retry_sleep: float,
|
|
request_timeout: float,
|
|
) -> str:
|
|
params = {
|
|
"query": query,
|
|
"searchtype": "all",
|
|
"abstracts": "show",
|
|
"order": "-announced_date_first",
|
|
"size": min(max_results, 200),
|
|
"start": start,
|
|
}
|
|
url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"
|
|
request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.2"})
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=request_timeout) as response:
|
|
return response.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
|
|
raise
|
|
print(
|
|
f"arXiv search returned HTTP {exc.code}; retrying request "
|
|
f"{attempt + 1}/{retries} after backoff",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
|
|
if attempt >= retries:
|
|
raise
|
|
print(
|
|
f"arXiv search failed with {type(exc).__name__}; retrying "
|
|
f"{attempt + 1}/{retries} after backoff",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
time.sleep(retry_sleep * (attempt + 1))
|
|
raise RuntimeError("unreachable search retry state")
|
|
|
|
|
|
def fetch_search_query(
|
|
query: str,
|
|
from_date: str,
|
|
to_date: str,
|
|
per_query: int,
|
|
page_size: int,
|
|
sleep_seconds: float,
|
|
retries: int,
|
|
retry_sleep: float,
|
|
request_timeout: float,
|
|
) -> list[dict]:
|
|
records: list[dict] = []
|
|
start = 0
|
|
while start < per_query:
|
|
requested = min(page_size, per_query - start, 200)
|
|
html_text = fetch_search_page(
|
|
query,
|
|
start,
|
|
requested,
|
|
retries,
|
|
retry_sleep,
|
|
request_timeout,
|
|
)
|
|
parser = ArxivSearchParser()
|
|
parser.feed(html_text)
|
|
page_records = parser.records
|
|
if not page_records:
|
|
break
|
|
records.extend(record for record in page_records if from_date <= record["published"] <= to_date)
|
|
print(
|
|
f"searched arXiv HTML: {start + len(page_records)} records, "
|
|
f"{len(records)} inside window",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
start += len(page_records)
|
|
if len(page_records) < requested:
|
|
break
|
|
# Announced-date ordering can mix old cross-listed papers into a recent page,
|
|
# so the original submission date is not a safe early-stop signal.
|
|
time.sleep(sleep_seconds)
|
|
return records
|
|
|
|
|
|
def manifest_record(record: dict, topics: list[str], score: int, relevance: str, matched_queries: set[str]) -> dict:
|
|
sorted_queries = sorted(matched_queries)
|
|
return {
|
|
"id": record["id"],
|
|
"arxiv_id": record["id"],
|
|
"source": "arxiv",
|
|
"source_id": f"arxiv:{record['id']}",
|
|
"title": record["title"],
|
|
"url": record["url"],
|
|
"pdf_url": f"https://arxiv.org/pdf/{record['id']}",
|
|
"published": record["published"],
|
|
"updated": record["updated"],
|
|
"authors": record["authors"],
|
|
"categories": record["categories"],
|
|
"topics": topics,
|
|
"score": score,
|
|
"relevance": relevance,
|
|
"primary_query": sorted_queries[0] if sorted_queries else "",
|
|
"matched_queries": sorted_queries,
|
|
}
|
|
|
|
|
|
def parse_feed(xml_text: str) -> list[dict]:
|
|
root = ET.fromstring(xml_text)
|
|
records = []
|
|
for entry in root.findall("atom:entry", NS):
|
|
entry_id = entry.findtext("atom:id", default="", namespaces=NS)
|
|
title = normalize_space(entry.findtext("atom:title", default="", namespaces=NS))
|
|
summary = normalize_space(entry.findtext("atom:summary", default="", namespaces=NS))
|
|
published = entry.findtext("atom:published", default="", namespaces=NS)[:10]
|
|
updated = entry.findtext("atom:updated", default="", namespaces=NS)[:10]
|
|
authors = [
|
|
normalize_space(author.findtext("atom:name", default="", namespaces=NS))
|
|
for author in entry.findall("atom:author", NS)
|
|
]
|
|
categories = [
|
|
category.attrib.get("term", "")
|
|
for category in entry.findall("atom:category", NS)
|
|
if category.attrib.get("term")
|
|
]
|
|
records.append(
|
|
{
|
|
"id": arxiv_id(entry_id),
|
|
"url": f"https://arxiv.org/abs/{arxiv_id(entry_id)}",
|
|
"title": title,
|
|
"summary": summary,
|
|
"published": published,
|
|
"updated": updated,
|
|
"authors": authors,
|
|
"categories": categories,
|
|
}
|
|
)
|
|
return records
|
|
|
|
|
|
def classify(record: dict, matched_queries: set[str]) -> tuple[list[str], int, str]:
|
|
text = f"{record['title']} {record['summary']}".lower()
|
|
title = record["title"].lower()
|
|
topics = []
|
|
score = 0
|
|
|
|
if "agent" in title or "agentic" in title:
|
|
score += 4
|
|
elif "agent" in text or "agentic" in text:
|
|
score += 2
|
|
|
|
if "llm" in text or "large language model" in text or "language model" in text:
|
|
score += 2
|
|
|
|
for term in HIGH_SIGNAL_TERMS:
|
|
if term in title:
|
|
score += 3
|
|
elif term in text:
|
|
score += 1
|
|
|
|
for topic, keywords in TOPIC_RULES:
|
|
if any(keyword in text for keyword in keywords):
|
|
topics.append(topic)
|
|
score += 1
|
|
|
|
if matched_queries:
|
|
score += min(len(matched_queries), 4)
|
|
|
|
if not topics and ("agent" in text or "agentic" in text):
|
|
topics.append("agent")
|
|
|
|
if score >= 13:
|
|
relevance = "high"
|
|
elif score >= 7:
|
|
relevance = "medium"
|
|
else:
|
|
relevance = "low"
|
|
|
|
return sorted(set(topics)), score, relevance
|
|
|
|
|
|
def existing_arxiv_ids() -> set[str]:
|
|
ids = set()
|
|
for path in (ROOT / "papers" / "items").glob("*.md"):
|
|
text = path.read_text(encoding="utf-8")
|
|
for match in re.findall(r"https://arxiv\.org/(?:abs|html)/([0-9]+\.[0-9]+)", text):
|
|
ids.add(match)
|
|
return ids
|
|
|
|
|
|
def item_path(record: dict) -> Path:
|
|
year = (record.get("published") or "0000")[:4]
|
|
title_slug = slugify(record["title"])
|
|
id_slug = record["id"].replace(".", "-")
|
|
return ROOT / "papers" / "items" / f"{year}-{id_slug}-{title_slug}.md"
|
|
|
|
|
|
def write_item(record: dict, topics: list[str], score: int, relevance: str, matched_queries: set[str], today: str) -> Path:
|
|
path = item_path(record)
|
|
authors = ", ".join(record["authors"][:8])
|
|
if len(record["authors"]) > 8:
|
|
authors += ", et al."
|
|
categories = ", ".join(record["categories"])
|
|
query_list = ", ".join(sorted(matched_queries))
|
|
topic_block = "\n".join(f" - {topic}" for topic in topics) or " - agent"
|
|
category_block = "\n".join(f" - {category}" for category in record["categories"]) or " -"
|
|
content = f"""# Paper: {record['title']}
|
|
|
|
---
|
|
type: paper
|
|
title: {yaml_value(record['title'])}
|
|
authors: {yaml_value(authors)}
|
|
year: {yaml_value((record.get('published') or '')[:4])}
|
|
venue: arXiv
|
|
url: {record['url']}
|
|
code_url:
|
|
source: arxiv
|
|
collected_at: {today}
|
|
published_at: {record.get('published') or ''}
|
|
updated_at: {record.get('updated') or ''}
|
|
status: queued
|
|
relevance: {relevance}
|
|
topics:
|
|
{topic_block}
|
|
methods:
|
|
-
|
|
benchmarks:
|
|
-
|
|
models:
|
|
-
|
|
datasets:
|
|
{category_block}
|
|
related_concepts:
|
|
-
|
|
related_jobs:
|
|
-
|
|
related_experiments:
|
|
-
|
|
related_projects:
|
|
-
|
|
collection_score: {score}
|
|
collection_queries: {yaml_value(query_list)}
|
|
---
|
|
|
|
## One-line Takeaway
|
|
|
|
Auto-collected from arXiv because it matched the Agent collection queries. Needs human skim.
|
|
|
|
## Why Collected
|
|
|
|
- matched queries: {query_list or 'agent-related arXiv sweep'}
|
|
- inferred topics: {', '.join(topics) or 'agent'}
|
|
- arXiv categories: {categories or 'unknown'}
|
|
- collection score: {score}
|
|
|
|
## Review Checklist
|
|
|
|
- Does this paper directly inform Agent architecture, evaluation, memory, tools, safety, coding agents, GUI/browser agents, or multi-agent workflows?
|
|
- Does it include a benchmark, dataset, code, or reproducible experimental setup?
|
|
- Should it be promoted from `queued` to `skimmed` or `summarized`?
|
|
|
|
## Links
|
|
|
|
- arXiv: {record['url']}
|
|
"""
|
|
path.write_text(content, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--from-date", default="2025-07-08")
|
|
parser.add_argument("--to-date", default=date.today().isoformat())
|
|
parser.add_argument("--per-query", type=int, default=120)
|
|
parser.add_argument("--page-size", type=int, default=60)
|
|
parser.add_argument("--max-items", type=int, default=180)
|
|
parser.add_argument("--min-score", type=int, default=6)
|
|
parser.add_argument("--sleep", type=float, default=1.0)
|
|
parser.add_argument("--retries", type=int, default=4)
|
|
parser.add_argument("--retry-sleep", type=float, default=10.0)
|
|
parser.add_argument("--request-timeout", type=float, default=60.0)
|
|
parser.add_argument("--backend", choices=("api", "search-html"), default="api")
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
by_id: dict[str, dict] = {}
|
|
matches: dict[str, set[str]] = defaultdict(set)
|
|
|
|
for label, raw_query in QUERIES:
|
|
if args.backend == "search-html":
|
|
print(f"collecting {label} from arXiv search", file=sys.stderr, flush=True)
|
|
records = fetch_search_query(
|
|
SEARCH_QUERIES[label],
|
|
args.from_date,
|
|
args.to_date,
|
|
args.per_query,
|
|
args.page_size,
|
|
args.sleep,
|
|
args.retries,
|
|
args.retry_sleep,
|
|
args.request_timeout,
|
|
)
|
|
for record in records:
|
|
by_id.setdefault(record["id"], record)
|
|
matches[record["id"]].add(label)
|
|
continue
|
|
fetched = 0
|
|
start = 0
|
|
print(f"collecting {label}", file=sys.stderr, flush=True)
|
|
while fetched < args.per_query:
|
|
page_size = min(args.page_size, args.per_query - fetched)
|
|
xml_text = fetch_query(
|
|
raw_query,
|
|
args.from_date,
|
|
args.to_date,
|
|
start,
|
|
page_size,
|
|
args.retries,
|
|
args.retry_sleep,
|
|
args.request_timeout,
|
|
)
|
|
records = parse_feed(xml_text)
|
|
if not records:
|
|
break
|
|
for record in records:
|
|
by_id.setdefault(record["id"], record)
|
|
matches[record["id"]].add(label)
|
|
fetched += len(records)
|
|
start += len(records)
|
|
print(
|
|
f"collected {label}: {fetched} records",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
if len(records) < page_size:
|
|
break
|
|
time.sleep(args.sleep)
|
|
|
|
existing = existing_arxiv_ids()
|
|
scored = []
|
|
all_candidates = []
|
|
for record_id, record in by_id.items():
|
|
topics, score, relevance = classify(record, matches[record_id])
|
|
all_candidates.append(manifest_record(record, topics, score, relevance, matches[record["id"]]))
|
|
if score < args.min_score:
|
|
continue
|
|
scored.append((score, relevance, topics, record))
|
|
|
|
scored.sort(key=lambda item: (item[0], item[3].get("published") or ""), reverse=True)
|
|
selected = scored[: args.max_items]
|
|
|
|
data_dir = ROOT / "data"
|
|
data_dir.mkdir(exist_ok=True)
|
|
manifest = []
|
|
written = []
|
|
skipped_existing = 0
|
|
today = date.today().isoformat()
|
|
for score, relevance, topics, record in selected:
|
|
manifest.append(manifest_record(record, topics, score, relevance, matches[record["id"]]))
|
|
if record["id"] in existing:
|
|
skipped_existing += 1
|
|
continue
|
|
if not args.dry_run:
|
|
path = write_item(record, topics, score, relevance, matches[record["id"]], today)
|
|
written.append(str(path.relative_to(ROOT)))
|
|
|
|
all_candidates.sort(key=lambda item: (item["score"], item["published"]), reverse=True)
|
|
candidates_path = data_dir / f"arxiv-agent-candidates-{args.from_date}-to-{args.to_date}.json"
|
|
candidates_path.write_text(json.dumps(all_candidates, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
manifest_path = data_dir / f"arxiv-agent-papers-{args.from_date}-to-{args.to_date}.json"
|
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
summary = {
|
|
"from_date": args.from_date,
|
|
"to_date": args.to_date,
|
|
"queries": [label for label, _ in QUERIES],
|
|
"unique_seen": len(by_id),
|
|
"selected": len(selected),
|
|
"written": len(written),
|
|
"skipped_existing": skipped_existing,
|
|
"candidates_manifest": str(candidates_path.relative_to(ROOT)),
|
|
"manifest": str(manifest_path.relative_to(ROOT)),
|
|
"written_paths": written,
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|