zk-data-agent/benchmarks/run_suite.py

#!/usr/bin/env python3
"""
Standard evaluation benchmark runner for claw-code-agent.

Runs the agent against well-known evaluation suites and reports scores.

Usage:
    # List available benchmark suites
    python3 -m benchmarks.run_suite --list

    # Run a specific suite (built-in subset)
    python3 -m benchmarks.run_suite --suite humaneval
    python3 -m benchmarks.run_suite --suite mbpp
    python3 -m benchmarks.run_suite --suite swe-bench
    python3 -m benchmarks.run_suite --suite aider
    python3 -m benchmarks.run_suite --suite livecodebench
    python3 -m benchmarks.run_suite --suite math
    python3 -m benchmarks.run_suite --suite gsm8k
    python3 -m benchmarks.run_suite --suite aime
    python3 -m benchmarks.run_suite --suite ifeval
    python3 -m benchmarks.run_suite --suite bfcl

    # Run ALL suites
    python3 -m benchmarks.run_suite --all

    # Run by category
    python3 -m benchmarks.run_suite --category coding
    python3 -m benchmarks.run_suite --category math
    python3 -m benchmarks.run_suite --category instruction-following

    # Limit problems per suite (for quick testing)
    python3 -m benchmarks.run_suite --suite humaneval --limit 5

    # Verbose output + save results
    python3 -m benchmarks.run_suite --suite humaneval -v -o results.json
"""

from __future__ import annotations

import argparse
import json
import sys
import time

from benchmarks.suites.base import BenchmarkSuite, SuiteReport

# Import all suites
from benchmarks.suites.humaneval import HumanEvalBenchmark
from benchmarks.suites.mbpp import MBPPBenchmark
from benchmarks.suites.swe_bench import SWEBenchBenchmark
from benchmarks.suites.aider import AiderBenchmark
from benchmarks.suites.livecodebench import LiveCodeBenchBenchmark
from benchmarks.suites.math_bench import MATHBenchmark
from benchmarks.suites.gsm8k import GSM8KBenchmark
from benchmarks.suites.aime import AIMEBenchmark
from benchmarks.suites.ifeval import IFEvalBenchmark
from benchmarks.suites.bfcl import BFCLBenchmark
from benchmarks.suites.mmlu_pro import MMLUProBenchmark
from benchmarks.suites.gpqa import GPQABenchmark
from benchmarks.suites.bigbench import BigBenchHardBenchmark
from benchmarks.suites.mmmlu import MMMMLUBenchmark
from benchmarks.suites.hle import HLEBenchmark
from benchmarks.suites.tau2 import Tau2Benchmark
from benchmarks.suites.codeforces import CodeforcesBenchmark


# ---------------------------------------------------------------------------
# Registry
# ---------------------------------------------------------------------------

SUITE_REGISTRY: dict[str, type[BenchmarkSuite]] = {
    "humaneval": HumanEvalBenchmark,
    "mbpp": MBPPBenchmark,
    "swe-bench": SWEBenchBenchmark,
    "aider": AiderBenchmark,
    "livecodebench": LiveCodeBenchBenchmark,
    "math": MATHBenchmark,
    "gsm8k": GSM8KBenchmark,
    "aime": AIMEBenchmark,
    "ifeval": IFEvalBenchmark,
    "bfcl": BFCLBenchmark,
    "mmlu-pro": MMLUProBenchmark,
    "gpqa-diamond": GPQABenchmark,
    "bigbench-hard": BigBenchHardBenchmark,
    "mmmlu": MMMMLUBenchmark,
    "hle": HLEBenchmark,
    "tau2": Tau2Benchmark,
    "codeforces": CodeforcesBenchmark,
}

CATEGORY_MAP: dict[str, list[str]] = {
    "coding": ["humaneval", "mbpp", "swe-bench", "aider", "livecodebench", "codeforces"],
    "math": ["math", "gsm8k", "aime"],
    "instruction-following": ["ifeval", "bfcl"],
    "knowledge": ["mmlu-pro", "gpqa-diamond", "mmmlu", "hle"],
    "reasoning": ["bigbench-hard", "tau2"],
}


# ---------------------------------------------------------------------------
# Multi-suite reporting
# ---------------------------------------------------------------------------

def print_combined_report(reports: list[SuiteReport]) -> None:
    """Print a combined summary of all suite runs."""
    print()
    print("=" * 80)
    print("  COMBINED BENCHMARK REPORT")
    print("=" * 80)
    print()
    print(f"  {'Suite':<20} {'Category':<22} {'Passed':>8} {'Total':>8} {'Score':>8} {'Time':>8}")
    print(f"  {'─' * 20} {'─' * 22} {'─' * 8} {'─' * 8} {'─' * 8} {'─' * 8}")

    total_passed = 0
    total_problems = 0
    total_time = 0.0

    for r in reports:
        suite_cls = SUITE_REGISTRY.get(r.suite_name.lower().replace("-", "").replace("_", ""))
        cat = suite_cls.category if suite_cls else "unknown"
        print(
            f"  {r.suite_name:<20} {cat:<22} "
            f"{r.passed:>8} {r.total:>8} "
            f"{r.score_pct:>7.1f}% {r.duration_sec:>7.1f}s"
        )
        total_passed += r.passed
        total_problems += r.total
        total_time += r.duration_sec

    print()
    print("─" * 80)
    overall_pct = round(100.0 * total_passed / total_problems, 1) if total_problems else 0.0
    print(
        f"  OVERALL: {total_passed}/{total_problems} ({overall_pct}%) "
        f"in {total_time:.1f}s"
    )
    print("─" * 80)

    # Category breakdown
    print()
    print("  Category Breakdown:")
    for cat_name, suite_names in CATEGORY_MAP.items():
        cat_reports = [r for r in reports if r.suite_name.lower().replace("-", "").replace("_", "") in
                       [s.replace("-", "").replace("_", "") for s in suite_names]]
        if cat_reports:
            cp = sum(r.passed for r in cat_reports)
            ct = sum(r.total for r in cat_reports)
            cpct = round(100.0 * cp / ct, 1) if ct else 0.0
            bar_len = 20
            filled = round(bar_len * cp / ct) if ct else 0
            bar = "█" * filled + "░" * (bar_len - filled)
            print(f"    {cat_name:<22} {bar}  {cp}/{ct} ({cpct}%)")

    print()


def save_combined_report(reports: list[SuiteReport], path: str) -> None:
    data = {
        "benchmark_run": "claw-code-agent-evaluation",
        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
        "suites": [r.to_dict() for r in reports],
        "summary": {
            "total_suites": len(reports),
            "total_problems": sum(r.total for r in reports),
            "total_passed": sum(r.passed for r in reports),
            "overall_score_pct": round(
                100.0 * sum(r.passed for r in reports) / sum(r.total for r in reports), 1
            ) if any(r.total for r in reports) else 0.0,
        },
    }
    with open(path, "w") as fh:
        json.dump(data, fh, indent=2)
    print(f"  Combined results saved to {path}")


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        description="Run standard evaluation benchmarks against claw-code-agent",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=(
            "Examples:\n"
            "  python3 -m benchmarks.run_suite --list\n"
            "  python3 -m benchmarks.run_suite --suite humaneval\n"
            "  python3 -m benchmarks.run_suite --suite humaneval --limit 5 -v\n"
            "  python3 -m benchmarks.run_suite --category coding\n"
            "  python3 -m benchmarks.run_suite --all\n"
            "  python3 -m benchmarks.run_suite --all -o results.json\n"
        ),
    )
    parser.add_argument("--suite", action="append", default=[],
                        help="Run a specific benchmark suite (can be repeated)")
    parser.add_argument("--category", choices=list(CATEGORY_MAP.keys()),
                        help="Run all suites in a category")
    parser.add_argument("--all", action="store_true",
                        help="Run ALL benchmark suites")
    parser.add_argument("--list", action="store_true",
                        help="List available benchmark suites")
    parser.add_argument("--limit", type=int, default=None,
                        help="Max problems per suite (for quick testing)")
    parser.add_argument("--timeout", type=float, default=300.0,
                        help="Agent timeout per problem in seconds (default: 300)")
    parser.add_argument("--verbose", "-v", action="store_true",
                        help="Verbose output")
    parser.add_argument("--output", "-o",
                        help="Save results to JSON file")
    parser.add_argument("--data-dir",
                        help="Directory containing dataset files (JSONL)")
    parser.add_argument("--artifacts-dir",
                        help="Directory where per-problem artifacts will be saved")
    parser.add_argument("--save-passing-artifacts", action="store_true",
                        help="Also save artifacts for passing problems")
    args = parser.parse_args()

    if args.list:
        print()
        print("  Available Benchmark Suites:")
        print("  " + "─" * 68)
        print(f"  {'Name':<18} {'Category':<22} {'Description'}")
        print("  " + "─" * 68)
        for name, cls in SUITE_REGISTRY.items():
            print(f"  {name:<18} {cls.category:<22} {cls.description}")
        print()
        print("  Categories:")
        for cat, suites in CATEGORY_MAP.items():
            print(f"    {cat}: {', '.join(suites)}")
        print()
        return

    # Determine which suites to run
    suite_names: list[str] = []
    if args.all:
        suite_names = list(SUITE_REGISTRY.keys())
    elif args.category:
        suite_names = CATEGORY_MAP.get(args.category, [])
    elif args.suite:
        for s in args.suite:
            if s not in SUITE_REGISTRY:
                print(f"Error: unknown suite '{s}'. Available: {', '.join(SUITE_REGISTRY)}")
                sys.exit(1)
        suite_names = list(args.suite)
    else:
        parser.print_help()
        print("\nError: specify --suite, --category, or --all")
        sys.exit(1)

    if not suite_names:
        print("No suites matched.")
        sys.exit(1)

    # Run suites
    reports: list[SuiteReport] = []
    for name in suite_names:
        cls = SUITE_REGISTRY[name]
        suite = cls(
            data_dir=args.data_dir,
            limit=args.limit,
            agent_timeout=args.timeout,
            verbose=args.verbose,
            artifacts_dir=args.artifacts_dir,
            save_passing_artifacts=args.save_passing_artifacts,
        )
        report = suite.run_all()
        reports.append(report)

    # Combined report
    if len(reports) > 1:
        print_combined_report(reports)

    if args.output:
        save_combined_report(reports, args.output)


if __name__ == "__main__":
    main()