Expand arXiv paper corpus

2026-07-08 12:25:30 +08:00
parent b4af5e1fbc
commit 391b3a4732
980 changed files with 189464 additions and 58 deletions
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Promote arXiv manifest records into paper item notes."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import date
+from pathlib import Path
+
+from collect_arxiv import ROOT, existing_arxiv_ids, write_item
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("manifest", help="Path to an arXiv candidate manifest JSON file.")
+    parser.add_argument("--min-score", type=int, default=13)
+    parser.add_argument("--relevance", default="high")
+    parser.add_argument("--max-items", type=int, default=0, help="0 means no cap.")
+    parser.add_argument("--output-manifest", help="Optional JSON file for promoted records.")
+    parser.add_argument("--dry-run", action="store_true")
+    return parser.parse_args()
+
+
+def normalize_record(record: dict) -> dict:
+    arxiv_id = record.get("arxiv_id") or record["id"]
+    return {
+        "id": arxiv_id,
+        "url": record.get("url") or f"https://arxiv.org/abs/{arxiv_id}",
+        "title": record["title"],
+        "published": record.get("published") or "",
+        "updated": record.get("updated") or "",
+        "authors": record.get("authors") or [],
+        "categories": record.get("categories") or [],
+    }
+
+
+def main() -> int:
+    args = parse_args()
+    manifest_path = Path(args.manifest)
+    if not manifest_path.is_absolute():
+        manifest_path = ROOT / manifest_path
+
+    records = json.loads(manifest_path.read_text(encoding="utf-8"))
+    selected = [
+        record
+        for record in records
+        if int(record.get("score") or 0) >= args.min_score
+        and (not args.relevance or record.get("relevance") == args.relevance)
+    ]
+    selected.sort(key=lambda item: (int(item.get("score") or 0), item.get("published") or ""), reverse=True)
+    if args.max_items:
+        selected = selected[: args.max_items]
+
+    existing = existing_arxiv_ids()
+    today = date.today().isoformat()
+    written = []
+    skipped_existing = 0
+    for record in selected:
+        arxiv_id = record.get("arxiv_id") or record["id"]
+        if arxiv_id in existing:
+            skipped_existing += 1
+            continue
+        if not args.dry_run:
+            path = write_item(
+                normalize_record(record),
+                record.get("topics") or ["agent"],
+                int(record.get("score") or 0),
+                record.get("relevance") or "unknown",
+                set(record.get("matched_queries") or []),
+                today,
+            )
+            written.append(str(path.relative_to(ROOT)))
+            existing.add(arxiv_id)
+
+    if args.output_manifest and not args.dry_run:
+        output_path = Path(args.output_manifest)
+        if not output_path.is_absolute():
+            output_path = ROOT / output_path
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(json.dumps(selected, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    print(
+        json.dumps(
+            {
+                "manifest": str(manifest_path.relative_to(ROOT)),
+                "selected": len(selected),
+                "written": len(written),
+                "skipped_existing": skipped_existing,
+                "output_manifest": args.output_manifest or "",
+                "written_paths": written,
+            },
+            ensure_ascii=False,
+            indent=2,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())