wuyang
102 lines
3.4 KiB
Python
102 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Promote arXiv manifest records into paper item notes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
from collect_arxiv import ROOT, existing_arxiv_ids, write_item
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("manifest", help="Path to an arXiv candidate manifest JSON file.")
|
|
parser.add_argument("--min-score", type=int, default=13)
|
|
parser.add_argument("--relevance", default="high")
|
|
parser.add_argument("--max-items", type=int, default=0, help="0 means no cap.")
|
|
parser.add_argument("--output-manifest", help="Optional JSON file for promoted records.")
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def normalize_record(record: dict) -> dict:
|
|
arxiv_id = record.get("arxiv_id") or record["id"]
|
|
return {
|
|
"id": arxiv_id,
|
|
"url": record.get("url") or f"https://arxiv.org/abs/{arxiv_id}",
|
|
"title": record["title"],
|
|
"published": record.get("published") or "",
|
|
"updated": record.get("updated") or "",
|
|
"authors": record.get("authors") or [],
|
|
"categories": record.get("categories") or [],
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
manifest_path = Path(args.manifest)
|
|
if not manifest_path.is_absolute():
|
|
manifest_path = ROOT / manifest_path
|
|
|
|
records = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
selected = [
|
|
record
|
|
for record in records
|
|
if int(record.get("score") or 0) >= args.min_score
|
|
and (not args.relevance or record.get("relevance") == args.relevance)
|
|
]
|
|
selected.sort(key=lambda item: (int(item.get("score") or 0), item.get("published") or ""), reverse=True)
|
|
if args.max_items:
|
|
selected = selected[: args.max_items]
|
|
|
|
existing = existing_arxiv_ids()
|
|
today = date.today().isoformat()
|
|
written = []
|
|
skipped_existing = 0
|
|
for record in selected:
|
|
arxiv_id = record.get("arxiv_id") or record["id"]
|
|
if arxiv_id in existing:
|
|
skipped_existing += 1
|
|
continue
|
|
if not args.dry_run:
|
|
path = write_item(
|
|
normalize_record(record),
|
|
record.get("topics") or ["agent"],
|
|
int(record.get("score") or 0),
|
|
record.get("relevance") or "unknown",
|
|
set(record.get("matched_queries") or []),
|
|
today,
|
|
)
|
|
written.append(str(path.relative_to(ROOT)))
|
|
existing.add(arxiv_id)
|
|
|
|
if args.output_manifest and not args.dry_run:
|
|
output_path = Path(args.output_manifest)
|
|
if not output_path.is_absolute():
|
|
output_path = ROOT / output_path
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
output_path.write_text(json.dumps(selected, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"manifest": str(manifest_path.relative_to(ROOT)),
|
|
"selected": len(selected),
|
|
"written": len(written),
|
|
"skipped_existing": skipped_existing,
|
|
"output_manifest": args.output_manifest or "",
|
|
"written_paths": written,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|