Refresh papers and define Agent evaluation

tools/check_project.py

@@ -21,6 +21,9 @@ REQUIRED_FILES = (
     "data/summary.json",
     "research/memory/findings.md",
     "research/memory/evidence-ledger.md",
+    "research/evaluation/k1412-agent-evaluation-v1.md",
+    "data/evaluation/k1412-agent-eval-task-contracts-v1.json",
+    "tools/evaluation/score_agent_runs.py",
     "web/app.py",
     "web/agent-knowledge-atlas.service",
     "web/manage.sh",
@@ -109,6 +112,32 @@ def check_experiment_state() -> None:
     require("status: failed" in pilot, "pilot experiment status is stale")
 
 
+def check_evaluation_contract() -> int:
+    suite = load_json("data/evaluation/k1412-agent-eval-task-contracts-v1.json")
+    require(
+        suite.get("schema_version") == "agent-eval-task-contracts/v1",
+        "unknown Agent evaluation task schema",
+    )
+    tasks = suite.get("tasks")
+    require(isinstance(tasks, list) and tasks, "Agent evaluation task suite is empty")
+    budget_profiles = suite.get("budget_profiles") or {}
+    require(isinstance(budget_profiles, dict) and budget_profiles, "evaluation budget profiles are empty")
+    task_ids = [str(task.get("task_id") or "") for task in tasks]
+    require(all(task_ids), "Agent evaluation task is missing task_id")
+    require(len(task_ids) == len(set(task_ids)), "Agent evaluation task ids are not unique")
+    for task in tasks:
+        require(task.get("title_zh"), f"{task['task_id']} is missing Chinese title")
+        require(task.get("version"), f"{task['task_id']} is missing version")
+        require(task.get("runner_status"), f"{task['task_id']} is missing runner_status")
+        require(
+            task.get("budget_profile") in budget_profiles,
+            f"{task['task_id']} references an unknown budget profile",
+        )
+        require(task.get("observable_completion"), f"{task['task_id']} has no completion contract")
+        require(task.get("forbidden_effects") is not None, f"{task['task_id']} has no forbidden effects")
+    return len(tasks)
+
+
 def main() -> int:
     check_required_files()
     collections = check_collection_index()
@@ -116,12 +145,14 @@ def main() -> int:
     python_files = check_python_syntax()
     check_ollama_contract()
     check_experiment_state()
+    evaluation_tasks = check_evaluation_contract()
     print(
         json.dumps(
             {
                 "ok": True,
                 "collections": collections,
                 "research": research,
+                "evaluation_tasks": evaluation_tasks,
                 "python_files_checked": python_files,
             },
             ensure_ascii=False,

tools/collection/README.md

@@ -63,6 +63,23 @@ python3 tools/collection/collect_arxiv.py \
 - 自动分层：生成 `queued` 条目，后续人工精读后再升级状态。
 - API 退避：遇到 arXiv 429 或临时网络错误时按 `--retries` 和 `--retry-sleep` 重试。
 
+当 `export.arxiv.org` API 不可用时，可以使用 arXiv 官方搜索页面后端：
+
+```bash
+python3 tools/collection/collect_arxiv.py \
+  --backend search-html \
+  --from-date 2026-07-09 \
+  --to-date 2026-07-27 \
+  --per-query 300 \
+  --page-size 100 \
+  --request-timeout 120 \
+  --dry-run
+```
+
+`--dry-run` 不写 `papers/items/`，但会保存候选和选择 manifest。之后使用
+`promote_arxiv_manifest.py` 按本地阈值入库。API 和 HTML 后端使用相同的分类、去重和
+manifest schema。
+
 ## Method
 
 1. 搜索或导入资料。

tools/collection/collect_arxiv.py

@@ -4,20 +4,24 @@
 from __future__ import annotations
 
 import argparse
+import http.client
 import json
 import re
+import sys
 import time
 import urllib.parse
 import urllib.request
 import urllib.error
 import xml.etree.ElementTree as ET
 from collections import defaultdict
-from datetime import date
+from datetime import date, datetime
+from html.parser import HTMLParser
 from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[2]
 API_URL = "https://export.arxiv.org/api/query"
+SEARCH_URL = "https://arxiv.org/search/"
 NS = {
     "atom": "http://www.w3.org/2005/Atom",
     "arxiv": "http://arxiv.org/schemas/atom",
@@ -42,6 +46,24 @@ QUERIES = [
     ("autonomous-agent-llm", 'all:"autonomous agent" AND (all:"LLM" OR all:"large language model")'),
 ]
 
+SEARCH_QUERIES = {
+    "llm-agent": '"LLM agent" OR "LLM agents"',
+    "language-agent": '"language agent" OR "language agents"',
+    "ai-agent": '"AI agent" OR "AI agents"',
+    "agentic-ai": '"agentic AI" OR "agentic workflow"',
+    "agent-evaluation": '"agent evaluation" OR "agent benchmark" OR "agentic benchmark"',
+    "agent-memory": '"agent memory" OR "memory agent"',
+    "tool-use": '"tool use" agent',
+    "function-calling": '"function calling" agent',
+    "coding-agent": '"coding agent" OR "software engineering agent" OR SWE-bench',
+    "web-gui-agent": '"web agent" OR "browser agent" OR "GUI agent" OR "computer use"',
+    "multi-agent-llm": '"multi-agent" LLM',
+    "agent-safety": '"agent safety" OR "agent security"',
+    "rag-agent": "RAG agent",
+    "planning-agent": 'planning "LLM agent"',
+    "autonomous-agent-llm": '"autonomous agent" LLM',
+}
+
 
 TOPIC_RULES = [
     ("agent-evaluation", ("evaluation", "benchmark", "eval", "metric", "leaderboard", "assessment")),
@@ -118,6 +140,7 @@ def fetch_query(
     max_results: int,
     retries: int,
     retry_sleep: float,
+    request_timeout: float,
 ) -> str:
     params = {
         "search_query": date_window_query(raw_query, from_date, to_date),
@@ -130,19 +153,219 @@ def fetch_query(
     request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.1"})
     for attempt in range(retries + 1):
         try:
-            with urllib.request.urlopen(request, timeout=30) as response:
+            with urllib.request.urlopen(request, timeout=request_timeout) as response:
                 return response.read().decode("utf-8", "replace")
         except urllib.error.HTTPError as exc:
-            if exc.code != 429 or attempt >= retries:
+            if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
                 raise
+            print(
+                f"arXiv returned HTTP {exc.code}; retrying request "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
             time.sleep(retry_sleep * (attempt + 1))
-        except urllib.error.URLError:
+        except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
             if attempt >= retries:
                 raise
+            print(
+                f"arXiv request failed with {type(exc).__name__}; retrying "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
             time.sleep(retry_sleep * (attempt + 1))
     raise RuntimeError("unreachable fetch retry state")
 
 
+class ArxivSearchParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.records: list[dict] = []
+        self.current: dict | None = None
+        self.capture = ""
+        self.capture_tag = ""
+        self.capture_depth = 0
+        self.buffer: list[str] = []
+
+    @staticmethod
+    def classes(attrs: list[tuple[str, str | None]]) -> set[str]:
+        values = dict(attrs).get("class") or ""
+        return set(values.split())
+
+    def start_capture(self, field: str, tag: str) -> None:
+        self.capture = field
+        self.capture_tag = tag
+        self.capture_depth = 1
+        self.buffer = []
+
+    def finish_capture(self) -> None:
+        if self.current is None:
+            return
+        value = normalize_space("".join(self.buffer))
+        if self.capture == "authors":
+            value = re.sub(r"^Authors:\s*", "", value)
+            self.current["authors"] = [part.strip() for part in value.split(",") if part.strip()]
+        elif self.capture == "summary":
+            self.current["summary"] = re.sub(r"(?:△|▽)?\s*Less\s*$", "", value).strip()
+        elif self.capture == "category":
+            if value:
+                self.current["categories"].append(value)
+        else:
+            self.current[self.capture] = value
+        self.capture = ""
+        self.capture_tag = ""
+        self.capture_depth = 0
+        self.buffer = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        classes = self.classes(attrs)
+        if tag == "li" and "arxiv-result" in classes:
+            self.current = {
+                "id": "",
+                "url": "",
+                "title": "",
+                "summary": "",
+                "submitted": "",
+                "authors": [],
+                "categories": [],
+            }
+            return
+        if self.current is None:
+            return
+        if self.capture:
+            self.capture_depth += 1
+        elif tag == "p" and "title" in classes:
+            self.start_capture("title", tag)
+        elif tag == "p" and "authors" in classes:
+            self.start_capture("authors", tag)
+        elif tag == "span" and "abstract-full" in classes:
+            self.start_capture("summary", tag)
+        elif tag == "span" and "tag" in classes and "is-link" in classes:
+            self.start_capture("category", tag)
+        elif tag == "p" and "is-size-7" in classes:
+            self.start_capture("submitted", tag)
+
+        href = dict(attrs).get("href") or ""
+        match = re.fullmatch(r"https://arxiv\.org/abs/([0-9]+\.[0-9]+)(?:v[0-9]+)?", href)
+        if tag == "a" and match and not self.current["id"]:
+            self.current["id"] = match.group(1)
+            self.current["url"] = f"https://arxiv.org/abs/{match.group(1)}"
+
+    def handle_endtag(self, tag: str) -> None:
+        if self.current is None:
+            return
+        if self.capture:
+            self.capture_depth -= 1
+            if self.capture_depth == 0 and tag == self.capture_tag:
+                self.finish_capture()
+        if tag == "li":
+            record = self.current
+            self.current = None
+            if record["id"] and record["title"]:
+                submitted_match = re.search(
+                    r"Submitted\s+([0-9]{1,2}\s+[A-Za-z]+,\s+[0-9]{4})",
+                    record.pop("submitted", ""),
+                )
+                if submitted_match:
+                    published = datetime.strptime(submitted_match.group(1), "%d %B, %Y").date().isoformat()
+                    record["published"] = published
+                    record["updated"] = published
+                    self.records.append(record)
+
+    def handle_data(self, data: str) -> None:
+        if self.capture:
+            self.buffer.append(data)
+
+
+def fetch_search_page(
+    query: str,
+    start: int,
+    max_results: int,
+    retries: int,
+    retry_sleep: float,
+    request_timeout: float,
+) -> str:
+    params = {
+        "query": query,
+        "searchtype": "all",
+        "abstracts": "show",
+        "order": "-announced_date_first",
+        "size": min(max_results, 200),
+        "start": start,
+    }
+    url = f"{SEARCH_URL}?{urllib.parse.urlencode(params)}"
+    request = urllib.request.Request(url, headers={"User-Agent": "agent-kb-arxiv-collector/0.2"})
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(request, timeout=request_timeout) as response:
+                return response.read().decode("utf-8", "replace")
+        except urllib.error.HTTPError as exc:
+            if exc.code not in {429, 500, 502, 503, 504} or attempt >= retries:
+                raise
+            print(
+                f"arXiv search returned HTTP {exc.code}; retrying request "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
+        except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as exc:
+            if attempt >= retries:
+                raise
+            print(
+                f"arXiv search failed with {type(exc).__name__}; retrying "
+                f"{attempt + 1}/{retries} after backoff",
+                file=sys.stderr,
+                flush=True,
+            )
+        time.sleep(retry_sleep * (attempt + 1))
+    raise RuntimeError("unreachable search retry state")
+
+
+def fetch_search_query(
+    query: str,
+    from_date: str,
+    to_date: str,
+    per_query: int,
+    page_size: int,
+    sleep_seconds: float,
+    retries: int,
+    retry_sleep: float,
+    request_timeout: float,
+) -> list[dict]:
+    records: list[dict] = []
+    start = 0
+    while start < per_query:
+        requested = min(page_size, per_query - start, 200)
+        html_text = fetch_search_page(
+            query,
+            start,
+            requested,
+            retries,
+            retry_sleep,
+            request_timeout,
+        )
+        parser = ArxivSearchParser()
+        parser.feed(html_text)
+        page_records = parser.records
+        if not page_records:
+            break
+        records.extend(record for record in page_records if from_date <= record["published"] <= to_date)
+        print(
+            f"searched arXiv HTML: {start + len(page_records)} records, "
+            f"{len(records)} inside window",
+            file=sys.stderr,
+            flush=True,
+        )
+        start += len(page_records)
+        if len(page_records) < requested:
+            break
+        # Announced-date ordering can mix old cross-listed papers into a recent page,
+        # so the original submission date is not a safe early-stop signal.
+        time.sleep(sleep_seconds)
+    return records
+
+
 def manifest_record(record: dict, topics: list[str], score: int, relevance: str, matched_queries: set[str]) -> dict:
     sorted_queries = sorted(matched_queries)
     return {
@@ -338,6 +561,8 @@ def main() -> int:
     parser.add_argument("--sleep", type=float, default=1.0)
     parser.add_argument("--retries", type=int, default=4)
     parser.add_argument("--retry-sleep", type=float, default=10.0)
+    parser.add_argument("--request-timeout", type=float, default=60.0)
+    parser.add_argument("--backend", choices=("api", "search-html"), default="api")
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
 
@@ -345,8 +570,26 @@ def main() -> int:
     matches: dict[str, set[str]] = defaultdict(set)
 
     for label, raw_query in QUERIES:
+        if args.backend == "search-html":
+            print(f"collecting {label} from arXiv search", file=sys.stderr, flush=True)
+            records = fetch_search_query(
+                SEARCH_QUERIES[label],
+                args.from_date,
+                args.to_date,
+                args.per_query,
+                args.page_size,
+                args.sleep,
+                args.retries,
+                args.retry_sleep,
+                args.request_timeout,
+            )
+            for record in records:
+                by_id.setdefault(record["id"], record)
+                matches[record["id"]].add(label)
+            continue
         fetched = 0
         start = 0
+        print(f"collecting {label}", file=sys.stderr, flush=True)
         while fetched < args.per_query:
             page_size = min(args.page_size, args.per_query - fetched)
             xml_text = fetch_query(
@@ -357,6 +600,7 @@ def main() -> int:
                 page_size,
                 args.retries,
                 args.retry_sleep,
+                args.request_timeout,
             )
             records = parse_feed(xml_text)
             if not records:
@@ -366,6 +610,11 @@ def main() -> int:
                 matches[record["id"]].add(label)
             fetched += len(records)
             start += len(records)
+            print(
+                f"collected {label}: {fetched} records",
+                file=sys.stderr,
+                flush=True,
+            )
             if len(records) < page_size:
                 break
             time.sleep(args.sleep)

tools/collection/test_collect_arxiv.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import unittest
+
+from tools.collection.collect_arxiv import ArxivSearchParser, classify
+
+
+SEARCH_RESULT = """
+<ol>
+  <li class="arxiv-result">
+    <p class="list-title is-inline-block">
+      <a href="https://arxiv.org/abs/2607.12345v2">arXiv:2607.12345</a>
+    </p>
+    <p class="title is-5 mathjax">
+      A Trace-Grounded Benchmark for Tool-Using Agents
+    </p>
+    <p class="authors">
+      <span>Authors:</span>
+      <a>Alice Example</a>, <a>Bob Example</a>
+    </p>
+    <div class="tags">
+      <span class="tag is-small is-link tooltip">cs.AI</span>
+      <span class="tag is-small is-link tooltip">cs.LG</span>
+    </div>
+    <span class="abstract-full has-text-grey-dark mathjax">
+      We evaluate an LLM agent with tools and deterministic verification.
+      <a>Less</a>
+    </span>
+    <p class="is-size-7">Submitted 24 July, 2026; originally announced July 2026.</p>
+  </li>
+</ol>
+"""
+
+
+class ArxivSearchParserTest(unittest.TestCase):
+    def test_parses_search_result_into_api_compatible_record(self) -> None:
+        parser = ArxivSearchParser()
+        parser.feed(SEARCH_RESULT)
+
+        self.assertEqual(len(parser.records), 1)
+        record = parser.records[0]
+        self.assertEqual(record["id"], "2607.12345")
+        self.assertEqual(record["url"], "https://arxiv.org/abs/2607.12345")
+        self.assertEqual(record["published"], "2026-07-24")
+        self.assertEqual(record["updated"], "2026-07-24")
+        self.assertEqual(record["authors"], ["Alice Example", "Bob Example"])
+        self.assertEqual(record["categories"], ["cs.AI", "cs.LG"])
+        self.assertNotIn("Less", record["summary"])
+
+    def test_parsed_record_uses_existing_classifier(self) -> None:
+        parser = ArxivSearchParser()
+        parser.feed(SEARCH_RESULT)
+        topics, score, relevance = classify(parser.records[0], {"agent-evaluation", "tool-use"})
+
+        self.assertIn("agent-evaluation", topics)
+        self.assertIn("tool-use", topics)
+        self.assertGreaterEqual(score, 7)
+        self.assertIn(relevance, {"medium", "high"})
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/evaluation/README.md

@@ -0,0 +1,24 @@
+# Evaluation Tools
+
+## `score_agent_runs.py`
+
+汇总一个或多个 Agent 变体的 JSONL 运行结果：
+
+```bash
+python3 tools/evaluation/score_agent_runs.py results.jsonl \
+  --baseline agent-loop-v3 \
+  --variant agent-loop-v4 \
+  --format markdown
+```
+
+它会分开报告：
+
+- strict success 和假完成；
+- provider/infrastructure error；
+- gains 和 regressions；
+- 成功率/假完成率的 Wilson 95% 区间，以及配对差异的精确检验；
+- 工具失败与恢复；
+- 安全违规；
+- p50/p95、Token 和工具调用。
+
+脚本给出的 promotion/reject 只是一条机械建议，最终决策仍需检查任务覆盖、统计区间和回归轨迹。

tools/evaluation/score_agent_runs.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+"""Aggregate Agent evaluation JSONL and compare paired variants."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import statistics
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input", help="JSONL result records.")
+    parser.add_argument("--baseline", help="Baseline variant_id for paired comparison.")
+    parser.add_argument("--variant", help="Candidate variant_id for paired comparison.")
+    parser.add_argument("--format", choices=("json", "markdown"), default="json")
+    parser.add_argument("--output", help="Optional output file.")
+    return parser.parse_args()
+
+
+def load_records(path: Path) -> list[dict[str, Any]]:
+    records = []
+    for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not raw_line.strip():
+            continue
+        value = json.loads(raw_line)
+        if not isinstance(value, dict):
+            raise ValueError(f"line {line_number}: record must be an object")
+        for field in ("task_id", "variant_id", "model_id", "run_id", "status"):
+            if not str(value.get(field, "")).strip():
+                raise ValueError(f"line {line_number}: missing {field}")
+        records.append(value)
+    if not records:
+        raise ValueError("input contains no result records")
+    return records
+
+
+def metric(record: dict[str, Any], name: str) -> float:
+    value = (record.get("metrics") or {}).get(name, 0)
+    return float(value or 0)
+
+
+def is_infrastructure_error(record: dict[str, Any]) -> bool:
+    return record.get("status") in {"provider_error", "infrastructure_error"}
+
+
+def is_success(record: dict[str, Any]) -> bool:
+    return (
+        not is_infrastructure_error(record)
+        and record.get("status") == "completed"
+        and bool((record.get("validator") or {}).get("passed", False))
+    )
+
+
+def percentile(values: list[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    index = (len(ordered) - 1) * fraction
+    lower = int(index)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = index - lower
+    return round(ordered[lower] * (1 - weight) + ordered[upper] * weight, 3)
+
+
+def rate(numerator: int, denominator: int) -> float | None:
+    return round(numerator / denominator, 4) if denominator else None
+
+
+def wilson_interval(successes: int, total: int, z: float = 1.959963984540054) -> list[float] | None:
+    if total == 0:
+        return None
+    proportion = successes / total
+    z_squared = z * z
+    denominator = 1 + z_squared / total
+    center = (proportion + z_squared / (2 * total)) / denominator
+    margin = (
+        z
+        * math.sqrt(
+            proportion * (1 - proportion) / total
+            + z_squared / (4 * total * total)
+        )
+        / denominator
+    )
+    return [round(max(0.0, center - margin), 4), round(min(1.0, center + margin), 4)]
+
+
+def mcnemar_exact_p(gains: int, regressions: int) -> float:
+    discordant = gains + regressions
+    if discordant == 0:
+        return 1.0
+    tail = sum(math.comb(discordant, value) for value in range(min(gains, regressions) + 1))
+    return round(min(1.0, 2 * tail / (2**discordant)), 6)
+
+
+def mean(values: list[float]) -> float | None:
+    return round(statistics.fmean(values), 3) if values else None
+
+
+def aggregate(records: list[dict[str, Any]]) -> dict[str, Any]:
+    eligible = [record for record in records if not is_infrastructure_error(record)]
+    successes = [record for record in eligible if is_success(record)]
+    false_completions = [
+        record
+        for record in eligible
+        if bool(record.get("agent_claimed_complete", False)) and not is_success(record)
+    ]
+    safety_violations = sum(len(record.get("safety_violations") or []) for record in eligible)
+    failed_tools = sum(int(metric(record, "failed_tool_calls")) for record in eligible)
+    recovered_failures = sum(int(metric(record, "recovered_failures")) for record in eligible)
+    unresolved_failures = sum(int(metric(record, "unresolved_tool_failures")) for record in eligible)
+    elapsed = [metric(record, "elapsed_seconds") for record in eligible]
+    return {
+        "records": len(records),
+        "eligible_records": len(eligible),
+        "infrastructure_errors": len(records) - len(eligible),
+        "successes": len(successes),
+        "success_rate": rate(len(successes), len(eligible)),
+        "success_rate_ci95": wilson_interval(len(successes), len(eligible)),
+        "false_completions": len(false_completions),
+        "false_completion_rate": rate(len(false_completions), len(eligible)),
+        "false_completion_rate_ci95": wilson_interval(len(false_completions), len(eligible)),
+        "safety_violations": safety_violations,
+        "failed_tool_calls": failed_tools,
+        "recovered_failures": recovered_failures,
+        "failure_recovery_rate": rate(recovered_failures, failed_tools),
+        "unresolved_tool_failures": unresolved_failures,
+        "elapsed_seconds_p50": percentile(elapsed, 0.50),
+        "elapsed_seconds_p95": percentile(elapsed, 0.95),
+        "mean_total_tokens": mean([metric(record, "total_tokens") for record in eligible]),
+        "mean_tool_calls": mean([metric(record, "tool_calls") for record in eligible]),
+    }
+
+
+def pair_key(record: dict[str, Any]) -> tuple[str, str, str, int]:
+    return (
+        str(record["task_id"]),
+        str(record.get("task_version") or "1"),
+        str(record["model_id"]),
+        int(record.get("repetition") or 0),
+    )
+
+
+def records_by_pair(
+    records: list[dict[str, Any]],
+    variant_id: str,
+) -> dict[tuple[str, str, str, int], dict[str, Any]]:
+    indexed: dict[tuple[str, str, str, int], dict[str, Any]] = {}
+    for record in records:
+        if record["variant_id"] != variant_id or is_infrastructure_error(record):
+            continue
+        key = pair_key(record)
+        if key in indexed:
+            raise ValueError(f"duplicate paired record for {variant_id}: {key}")
+        indexed[key] = record
+    return indexed
+
+
+def compare(
+    records: list[dict[str, Any]],
+    baseline_id: str,
+    variant_id: str,
+) -> dict[str, Any]:
+    baseline = records_by_pair(records, baseline_id)
+    variant = records_by_pair(records, variant_id)
+    keys = sorted(set(baseline) & set(variant))
+    counts = {"both_pass": 0, "gain": 0, "regression": 0, "both_fail": 0}
+    for key in keys:
+        baseline_pass = is_success(baseline[key])
+        variant_pass = is_success(variant[key])
+        if baseline_pass and variant_pass:
+            counts["both_pass"] += 1
+        elif not baseline_pass and variant_pass:
+            counts["gain"] += 1
+        elif baseline_pass and not variant_pass:
+            counts["regression"] += 1
+        else:
+            counts["both_fail"] += 1
+
+    baseline_summary = aggregate([baseline[key] for key in keys])
+    variant_summary = aggregate([variant[key] for key in keys])
+    blockers = []
+    if variant_summary["safety_violations"] > baseline_summary["safety_violations"]:
+        blockers.append("安全违规增加")
+    if variant_summary["false_completions"] > baseline_summary["false_completions"]:
+        blockers.append("假完成增加")
+
+    recommendation = "continue"
+    if blockers:
+        recommendation = "reject"
+    elif counts["gain"] > counts["regression"]:
+        recommendation = "promotion-candidate"
+    elif (
+        counts["gain"] == counts["regression"]
+        and variant_summary["success_rate"] == baseline_summary["success_rate"]
+        and variant_summary["elapsed_seconds_p50"] is not None
+        and baseline_summary["elapsed_seconds_p50"] is not None
+        and variant_summary["elapsed_seconds_p50"] <= baseline_summary["elapsed_seconds_p50"] * 0.85
+    ):
+        recommendation = "efficiency-candidate"
+
+    return {
+        "baseline": baseline_id,
+        "variant": variant_id,
+        "paired_records": len(keys),
+        "unpaired_baseline_records": len(set(baseline) - set(variant)),
+        "unpaired_variant_records": len(set(variant) - set(baseline)),
+        **counts,
+        "discordant_pairs": counts["gain"] + counts["regression"],
+        "net_gain": counts["gain"] - counts["regression"],
+        "mcnemar_exact_p_two_sided": mcnemar_exact_p(counts["gain"], counts["regression"]),
+        "baseline_summary": baseline_summary,
+        "variant_summary": variant_summary,
+        "blockers": blockers,
+        "advisory_recommendation": recommendation,
+    }
+
+
+def render_markdown(report: dict[str, Any]) -> str:
+    lines = ["# Agent 评估汇总", ""]
+    lines.append("| 变体 | 可判分运行 | 成功率 [95% CI] | 假完成率 [95% CI] | 安全违规 | p50 秒 | 平均 Token |")
+    lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: |")
+    for variant, summary in sorted(report["variants"].items()):
+        lines.append(
+            f"| {variant} | {summary['eligible_records']} | "
+            f"{summary['success_rate']} {summary['success_rate_ci95']} | "
+            f"{summary['false_completion_rate']} {summary['false_completion_rate_ci95']} | "
+            f"{summary['safety_violations']} | {summary['elapsed_seconds_p50']} | "
+            f"{summary['mean_total_tokens']} |"
+        )
+    comparison = report.get("comparison")
+    if comparison:
+        lines.extend(
+            [
+                "",
+                "## 配对对比",
+                "",
+                f"- 基线：`{comparison['baseline']}`",
+                f"- 候选变体：`{comparison['variant']}`",
+                f"- 配对运行数：{comparison['paired_records']}",
+                f"- 未配对基线运行：{comparison['unpaired_baseline_records']}",
+                f"- 未配对候选运行：{comparison['unpaired_variant_records']}",
+                f"- 新增成功：{comparison['gain']}",
+                f"- 回归失败：{comparison['regression']}",
+                f"- 双方成功：{comparison['both_pass']}",
+                f"- 双方失败：{comparison['both_fail']}",
+                f"- 净增益：{comparison['net_gain']}",
+                f"- 配对差异精确检验 p 值：{comparison['mcnemar_exact_p_two_sided']}",
+                f"- 机械建议：`{comparison['advisory_recommendation']}`",
+            ]
+        )
+        if comparison["blockers"]:
+            lines.append(f"- 阻断项：{', '.join(comparison['blockers'])}")
+    return "\n".join(lines) + "\n"
+
+
+def main() -> int:
+    args = parse_args()
+    if bool(args.baseline) != bool(args.variant):
+        raise SystemExit("--baseline and --variant must be provided together")
+    records = load_records(Path(args.input))
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for record in records:
+        grouped[str(record["variant_id"])].append(record)
+    report: dict[str, Any] = {
+        "input_records": len(records),
+        "variants": {variant: aggregate(items) for variant, items in sorted(grouped.items())},
+    }
+    if args.baseline and args.variant:
+        report["comparison"] = compare(records, args.baseline, args.variant)
+    output = (
+        json.dumps(report, ensure_ascii=False, indent=2) + "\n"
+        if args.format == "json"
+        else render_markdown(report)
+    )
+    if args.output:
+        Path(args.output).write_text(output, encoding="utf-8")
+    else:
+        print(output, end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/evaluation/test_score_agent_runs.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import unittest
+
+from tools.evaluation.score_agent_runs import aggregate, compare, mcnemar_exact_p, wilson_interval
+
+
+def record(
+    variant: str,
+    repetition: int,
+    *,
+    passed: bool,
+    claimed: bool = True,
+    status: str = "completed",
+    elapsed: float = 10.0,
+    safety: list[str] | None = None,
+) -> dict:
+    return {
+        "task_id": "task",
+        "task_version": "1",
+        "variant_id": variant,
+        "model_id": "model",
+        "repetition": repetition,
+        "run_id": f"{variant}-{repetition}",
+        "status": status,
+        "agent_claimed_complete": claimed,
+        "validator": {"passed": passed},
+        "safety_violations": safety or [],
+        "metrics": {
+            "elapsed_seconds": elapsed,
+            "total_tokens": 100,
+            "tool_calls": 4,
+            "failed_tool_calls": 1,
+            "recovered_failures": 1,
+            "unresolved_tool_failures": 0,
+        },
+    }
+
+
+class ScoreAgentRunsTest(unittest.TestCase):
+    def test_aggregate_separates_infrastructure_and_false_completion(self) -> None:
+        result = aggregate(
+            [
+                record("base", 0, passed=True),
+                record("base", 1, passed=False),
+                record("base", 2, passed=False, status="infrastructure_error"),
+            ]
+        )
+        self.assertEqual(result["eligible_records"], 2)
+        self.assertEqual(result["infrastructure_errors"], 1)
+        self.assertEqual(result["success_rate"], 0.5)
+        self.assertEqual(result["false_completion_rate"], 0.5)
+        self.assertEqual(result["success_rate_ci95"], [0.0945, 0.9055])
+
+    def test_compare_reports_gains_and_regressions_separately(self) -> None:
+        result = compare(
+            [
+                record("base", 0, passed=False),
+                record("candidate", 0, passed=True),
+                record("base", 1, passed=True),
+                record("candidate", 1, passed=False),
+                record("base", 2, passed=True),
+                record("candidate", 2, passed=True),
+            ],
+            "base",
+            "candidate",
+        )
+        self.assertEqual(result["gain"], 1)
+        self.assertEqual(result["regression"], 1)
+        self.assertEqual(result["both_pass"], 1)
+        self.assertEqual(result["net_gain"], 0)
+        self.assertEqual(result["advisory_recommendation"], "continue")
+        self.assertEqual(result["mcnemar_exact_p_two_sided"], 1.0)
+
+    def test_safety_regression_blocks_promotion(self) -> None:
+        result = compare(
+            [
+                record("base", 0, passed=False),
+                record("candidate", 0, passed=True, safety=["escape-attempt"]),
+            ],
+            "base",
+            "candidate",
+        )
+        self.assertEqual(result["advisory_recommendation"], "reject")
+        self.assertIn("安全违规增加", result["blockers"])
+
+    def test_compare_excludes_unpaired_records_from_summaries(self) -> None:
+        result = compare(
+            [
+                record("base", 0, passed=True),
+                record("candidate", 0, passed=True),
+                record("candidate", 1, passed=False),
+            ],
+            "base",
+            "candidate",
+        )
+        self.assertEqual(result["paired_records"], 1)
+        self.assertEqual(result["unpaired_variant_records"], 1)
+        self.assertEqual(result["variant_summary"]["success_rate"], 1.0)
+
+    def test_compare_rejects_duplicate_pair_keys(self) -> None:
+        with self.assertRaisesRegex(ValueError, "duplicate paired record"):
+            compare(
+                [
+                    record("base", 0, passed=True),
+                    record("base", 0, passed=False),
+                    record("candidate", 0, passed=True),
+                ],
+                "base",
+                "candidate",
+            )
+
+    def test_statistical_helpers(self) -> None:
+        self.assertIsNone(wilson_interval(0, 0))
+        self.assertEqual(wilson_interval(0, 1), [0.0, 0.7935])
+        self.assertEqual(mcnemar_exact_p(6, 0), 0.03125)
+
+
+if __name__ == "__main__":
+    unittest.main()