diff --git a/benchmarks/README.md b/benchmarks/README.md deleted file mode 100644 index 70d9415..0000000 --- a/benchmarks/README.md +++ /dev/null @@ -1,373 +0,0 @@ -# Benchmarks — claw-code-agent - -This directory contains two benchmark systems: - -1. **Local task benchmarks** (`benchmarks/run.py`) — custom tasks that test agent capabilities directly -2. **Standard evaluation suites** (`benchmarks/run_suite.py`) — implementations of well-known AI evaluation benchmarks - ---- - -## Quick Start - -```bash -# From the repository root: - -# Set up your model endpoint -export OPENAI_API_KEY="your-key" -export OPENAI_MODEL="gpt-4" # or your model name -export OPENAI_BASE_URL="http://localhost:8000/v1" # if using local vLLM/ollama - -# Run a quick smoke test (5 problems from HumanEval) -python3 -m benchmarks.run_suite --suite humaneval --limit 5 -v - -# Run all benchmarks -python3 -m benchmarks.run_suite --all -o results.json -``` - ---- - -## Standard Evaluation Suites - -### Available Benchmarks - -| Suite | Category | # Problems (built-in) | Description | -|-------|----------|----------------------|-------------| -| **HumanEval** | Coding | 20 | Code generation from Python docstrings | -| **MBPP** | Coding | 15 | Basic Python programming problems | -| **SWE-Bench** | Coding | 5 | Resolve real-world GitHub issues | -| **Aider** | Coding | 6 | Code editing and refactoring tasks | -| **LiveCodeBench** | Coding | 5 | Competitive programming problems | -| **Codeforces** | Coding | 10 | Competitive programming with ELO rating | -| **MATH** | Math | 15 | Competition mathematics problems | -| **GSM8K** | Math | 15 | Grade school math word problems | -| **AIME** | Math | 10 | Challenging competition math (integers 0–999) | -| **IFEval** | Instruction Following | 10 | Verifiable instruction-following evaluation | -| **BFCL** | Instruction Following | 7 | Function/tool calling evaluation | -| **MMLU-Pro** | Knowledge | 10 | Professional-level multiple-choice QA (10 choices) | -| **GPQA-Diamond** | Knowledge | 10 | Graduate-level science QA (diamond difficulty) | -| **MMMLU** | Knowledge | 10 | Multilingual MMLU across languages | -| **HLE** | Knowledge | 10 | Humanity's Last Exam (extremely hard) | -| **BigBench-Hard** | Reasoning | 10 | BIG-Bench Extra Hard reasoning tasks | -| **Tau2** | Reasoning | 10 | Tool-augmented reasoning (retail/airline/finance) | - -### Commands - -#### List All Suites - -```bash -python3 -m benchmarks.run_suite --list -``` - -#### Run a Specific Suite - -```bash -# Coding benchmarks -python3 -m benchmarks.run_suite --suite humaneval -python3 -m benchmarks.run_suite --suite mbpp -python3 -m benchmarks.run_suite --suite swe-bench -python3 -m benchmarks.run_suite --suite aider -python3 -m benchmarks.run_suite --suite livecodebench -python3 -m benchmarks.run_suite --suite codeforces - -# Math benchmarks -python3 -m benchmarks.run_suite --suite math -python3 -m benchmarks.run_suite --suite gsm8k -python3 -m benchmarks.run_suite --suite aime - -# Knowledge benchmarks -python3 -m benchmarks.run_suite --suite mmlu-pro -python3 -m benchmarks.run_suite --suite gpqa-diamond -python3 -m benchmarks.run_suite --suite mmmlu -python3 -m benchmarks.run_suite --suite hle - -# Reasoning benchmarks -python3 -m benchmarks.run_suite --suite bigbench-hard -python3 -m benchmarks.run_suite --suite tau2 - -# Instruction following benchmarks -python3 -m benchmarks.run_suite --suite ifeval -python3 -m benchmarks.run_suite --suite bfcl -``` - -#### Run by Category - -```bash -# All coding benchmarks -python3 -m benchmarks.run_suite --category coding - -# All math benchmarks -python3 -m benchmarks.run_suite --category math - -# All knowledge benchmarks (MMLU-Pro, GPQA, MMMLU, HLE) -python3 -m benchmarks.run_suite --category knowledge - -# All reasoning benchmarks (BigBench-Hard, Tau2) -python3 -m benchmarks.run_suite --category reasoning - -# All instruction following benchmarks -python3 -m benchmarks.run_suite --category instruction-following -``` - -#### Run ALL Suites - -```bash -python3 -m benchmarks.run_suite --all -``` - -#### Limit Problems (Quick Testing) - -```bash -# Run only first 3 problems from each suite -python3 -m benchmarks.run_suite --all --limit 3 - -# Quick HumanEval smoke test -python3 -m benchmarks.run_suite --suite humaneval --limit 5 -v -``` - -#### Save Results to JSON - -```bash -python3 -m benchmarks.run_suite --all -o results.json -python3 -m benchmarks.run_suite --suite humaneval -o humaneval_results.json -``` - -#### Verbose Output - -```bash -python3 -m benchmarks.run_suite --suite humaneval -v -``` - -#### Custom Timeout - -```bash -# 10 minutes per problem -python3 -m benchmarks.run_suite --suite swe-bench --timeout 600 -``` - -#### Use Full Datasets (JSONL) - -```bash -# Download or place your datasets in benchmarks/data/ -python3 -m benchmarks.run_suite --suite humaneval --data-dir ./benchmarks/data/ - -# Or specify a custom directory -python3 -m benchmarks.run_suite --suite humaneval --data-dir /path/to/datasets/ -``` - ---- - -## Using Full Datasets - -Each suite looks for a JSONL file in the data directory. If the file isn't found, it falls back to the built-in subset. - -| Suite | Expected File | Format | -|-------|--------------|--------| -| HumanEval | `humaneval.jsonl` | `{"task_id", "prompt", "canonical_solution", "test", "entry_point"}` | -| MBPP | `mbpp.jsonl` | `{"task_id", "text", "code", "test_list"}` | -| SWE-Bench | `swe_bench.jsonl` | `{"instance_id", "problem_statement", "setup_code", "test_cmd"}` | -| Aider | `aider.jsonl` | `{"id", "instruction", "setup_code", "test_code"}` | -| LiveCodeBench | `livecodebench.jsonl` | `{"id", "title", "description", "test_cases", "function_name"}` | -| Codeforces | `codeforces.jsonl` | `{"id", "rating", "title", "problem", "test_cases"}` | -| MATH | `math.jsonl` | `{"id", "problem", "answer", "subject", "level"}` | -| GSM8K | `gsm8k.jsonl` | `{"id", "question", "answer"}` | -| AIME | `aime.jsonl` or `aime_2026.jsonl` | `{"id", "problem", "answer"}` | -| IFEval | `ifeval.jsonl` | `{"id", "instruction", "checks"}` | -| BFCL | `bfcl.jsonl` | `{"id", "instruction", "expected_function", "setup_code", "test_code"}` | -| MMLU-Pro | `mmlu_pro.jsonl` | `{"id", "subject", "question", "choices", "answer"}` | -| GPQA-Diamond | `gpqa.jsonl` | `{"id", "subject", "question", "choices", "answer"}` | -| MMMLU | `mmmlu.jsonl` | `{"id", "language", "subject", "question", "choices", "answer"}` | -| HLE | `hle.jsonl` | `{"id", "subject", "question", "answer", "answer_type"}` | -| BigBench-Hard | `bigbench_hard.jsonl` | `{"id", "task", "question", "choices", "answer"}` | -| Tau2 | `tau2.jsonl` | `{"id", "domain", "question", "choices", "answer"}` | - -### Downloading Full Datasets - -```bash -# Download all datasets (builtin + official where available) -python3 -m benchmarks.download_datasets --all - -# Download builtin-only (no network, uses embedded problems) -python3 -m benchmarks.download_datasets --all --builtin-only - -# Download specific suites -python3 -m benchmarks.download_datasets --suite humaneval --suite mmlu-pro --suite gpqa-diamond - -# Force re-download -python3 -m benchmarks.download_datasets --all --force -``` - -**Datasets with HuggingFace downloaders:** -HumanEval, GSM8K, MBPP, MATH, MMLU-Pro, GPQA-Diamond, BigBench-Hard, MMMLU, HLE - -**Builtin-only datasets (no HuggingFace source):** -SWE-Bench, Aider, LiveCodeBench, AIME, IFEval, BFCL, Tau2, Codeforces - ---- - -## Local Task Benchmarks - -The original local benchmark system tests the agent on custom tasks. - -```bash -# Run all local tasks -python3 -m benchmarks.run - -# Run a single task -python3 -m benchmarks.run --task file-create-basic - -# Run a category -python3 -m benchmarks.run --category bugfix - -# Run by difficulty -python3 -m benchmarks.run --difficulty easy - -# List available tasks -python3 -m benchmarks.run --list - -# Verbose + save results -python3 -m benchmarks.run -v -o local_results.json -``` - ---- - -## All Commands Reference - -```bash -# ─── STANDARD EVALUATION SUITES ──────────────────────────────────────── - -# List suites -python3 -m benchmarks.run_suite --list - -# Individual suites -python3 -m benchmarks.run_suite --suite humaneval # Coding: docstring → code -python3 -m benchmarks.run_suite --suite mbpp # Coding: basic Python -python3 -m benchmarks.run_suite --suite swe-bench # Coding: GitHub issues -python3 -m benchmarks.run_suite --suite aider # Coding: code editing -python3 -m benchmarks.run_suite --suite livecodebench # Coding: competitive programming -python3 -m benchmarks.run_suite --suite codeforces # Coding: competitive + ELO -python3 -m benchmarks.run_suite --suite math # Math: competition math -python3 -m benchmarks.run_suite --suite gsm8k # Math: grade school -python3 -m benchmarks.run_suite --suite aime # Math: AIME competition -python3 -m benchmarks.run_suite --suite mmlu-pro # Knowledge: 14 subjects, 10 choices -python3 -m benchmarks.run_suite --suite gpqa-diamond # Knowledge: graduate science -python3 -m benchmarks.run_suite --suite mmmlu # Knowledge: multilingual MMLU -python3 -m benchmarks.run_suite --suite hle # Knowledge: Humanity's Last Exam -python3 -m benchmarks.run_suite --suite bigbench-hard # Reasoning: BIG-Bench Hard -python3 -m benchmarks.run_suite --suite tau2 # Reasoning: tool-augmented -python3 -m benchmarks.run_suite --suite ifeval # Instruction: format following -python3 -m benchmarks.run_suite --suite bfcl # Instruction: function calling - -# Category runs -python3 -m benchmarks.run_suite --category coding # All coding -python3 -m benchmarks.run_suite --category math # All math -python3 -m benchmarks.run_suite --category knowledge # MMLU-Pro, GPQA, MMMLU, HLE -python3 -m benchmarks.run_suite --category reasoning # BigBench, Tau2 -python3 -m benchmarks.run_suite --category instruction-following # IFEval, BFCL - -# Full run -python3 -m benchmarks.run_suite --all # All suites (~108 problems) -python3 -m benchmarks.run_suite --all --limit 3 # Quick: 3 per suite -python3 -m benchmarks.run_suite --all -v -o results.json # Full + verbose + save - -# Options -python3 -m benchmarks.run_suite --suite humaneval --limit 5 # Limit problems -python3 -m benchmarks.run_suite --suite humaneval --timeout 600 # Custom timeout (seconds) -python3 -m benchmarks.run_suite --suite humaneval --data-dir ./data # Custom data directory -python3 -m benchmarks.run_suite --suite humaneval -v # Verbose mode -python3 -m benchmarks.run_suite --suite humaneval -o out.json # Save to JSON - -# ─── LOCAL TASK BENCHMARKS ───────────────────────────────────────────── - -python3 -m benchmarks.run --list # List local tasks -python3 -m benchmarks.run # Run all local tasks -python3 -m benchmarks.run --task file-create-basic # Single task -python3 -m benchmarks.run --category bugfix # Category filter -python3 -m benchmarks.run --difficulty easy # Difficulty filter -python3 -m benchmarks.run -v -o local_results.json # Verbose + save -``` - ---- - -## Environment Variables - -| Variable | Description | Example | -|----------|-------------|---------| -| `OPENAI_API_KEY` | API key for the model provider | `sk-...` | -| `OPENAI_MODEL` | Model name to use | `gpt-4`, `qwen2.5-coder-32b` | -| `OPENAI_BASE_URL` | API base URL (for local models) | `http://localhost:8000/v1` | - ---- - -## Output Format - -Results are saved as JSON with this structure: - -```json -{ - "benchmark_run": "claw-code-agent-evaluation", - "timestamp": "2025-01-15T10:30:00", - "suites": [ - { - "suite_name": "HumanEval", - "total": 20, - "passed": 15, - "failed": 5, - "score_pct": 75.0, - "duration_sec": 450.5, - "model": "gpt-4", - "results": [ - { - "problem_id": "HumanEval/0", - "passed": true, - "duration_sec": 12.3, - "error": "" - } - ] - } - ], - "summary": { - "total_suites": 10, - "total_problems": 108, - "total_passed": 85, - "overall_score_pct": 78.7 - } -} -``` - ---- - -## Architecture - -``` -benchmarks/ -├── __init__.py -├── run.py # Local task benchmark runner -├── run_suite.py # Standard evaluation suite runner (CLI) -├── download_datasets.py # Dataset downloader (HuggingFace + builtins) -├── README.md # This file -├── data/ # Dataset files (JSONL) — not committed -│ └── .gitkeep -├── tasks/ -│ ├── __init__.py -│ └── definitions.py # Local task definitions -└── suites/ - ├── __init__.py - ├── base.py # Base class for all suites - ├── humaneval.py # HumanEval benchmark - ├── mbpp.py # MBPP benchmark - ├── swe_bench.py # SWE-Bench benchmark - ├── aider.py # Aider benchmark - ├── livecodebench.py # LiveCodeBench benchmark - ├── codeforces.py # Codeforces (ELO scoring) - ├── math_bench.py # MATH benchmark - ├── gsm8k.py # GSM8K benchmark - ├── aime.py # AIME benchmark - ├── mmlu_pro.py # MMLU-Pro (10-choice QA) - ├── gpqa.py # GPQA Diamond (science) - ├── mmmlu.py # MMMLU (multilingual) - ├── hle.py # HLE (Humanity's Last Exam) - ├── bigbench.py # BigBench Extra Hard - ├── tau2.py # Tau2 (tool-augmented) - ├── ifeval.py # IFEval benchmark - └── bfcl.py # BFCL benchmark -``` diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py deleted file mode 100644 index 1631abf..0000000 --- a/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Claw Code Agent local benchmark suite.""" diff --git a/benchmarks/data/.gitkeep b/benchmarks/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/download_datasets.py b/benchmarks/download_datasets.py deleted file mode 100644 index 20559ae..0000000 --- a/benchmarks/download_datasets.py +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env python3 -""" -Download or export benchmark datasets into benchmarks/data. - -Uses the HuggingFace `datasets` library for reliable full downloads. -Falls back to the REST API or builtins if `datasets` is not installed. -""" - -from __future__ import annotations - -import argparse -import gzip -import json -import re -import urllib.parse -import urllib.request -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Callable - -from benchmarks.suites.aider import _BUILTIN_PROBLEMS as _AIDER_BUILTINS -from benchmarks.suites.aime import _BUILTIN_PROBLEMS as _AIME_BUILTINS -from benchmarks.suites.bfcl import _BUILTIN_PROBLEMS as _BFCL_BUILTINS -from benchmarks.suites.bigbench import _BUILTIN_PROBLEMS as _BIGBENCH_BUILTINS -from benchmarks.suites.codeforces import _BUILTIN_PROBLEMS as _CODEFORCES_BUILTINS -from benchmarks.suites.gpqa import _BUILTIN_PROBLEMS as _GPQA_BUILTINS -from benchmarks.suites.gsm8k import _BUILTIN_PROBLEMS as _GSM8K_BUILTINS -from benchmarks.suites.hle import _BUILTIN_PROBLEMS as _HLE_BUILTINS -from benchmarks.suites.humaneval import _BUILTIN_PROBLEMS as _HUMANEVAL_BUILTINS -from benchmarks.suites.ifeval import _BUILTIN_PROBLEMS as _IFEVAL_BUILTINS -from benchmarks.suites.livecodebench import _BUILTIN_PROBLEMS as _LIVECODEBENCH_BUILTINS -from benchmarks.suites.math_bench import _BUILTIN_PROBLEMS as _MATH_BUILTINS -from benchmarks.suites.mbpp import _BUILTIN_PROBLEMS as _MBPP_BUILTINS -from benchmarks.suites.mmmlu import _BUILTIN_PROBLEMS as _MMMLU_BUILTINS -from benchmarks.suites.mmlu_pro import _BUILTIN_PROBLEMS as _MMLU_PRO_BUILTINS -from benchmarks.suites.swe_bench import _BUILTIN_PROBLEMS as _SWE_BUILTINS -from benchmarks.suites.tau2 import _BUILTIN_PROBLEMS as _TAU2_BUILTINS - - -HUMANEVAL_GZ_URL = "https://raw.githubusercontent.com/openai/human-eval/master/data/HumanEval.jsonl.gz" -DEFAULT_DATA_DIR = Path(__file__).resolve().parent / "data" - -# Legacy REST API support (fallback only) -HF_DATASET_VIEWER_BASE = "https://datasets-server.huggingface.co" -JsonFetcher = Callable[[str, dict[str, object], dict[str, str] | None, float], object] - - -@dataclass -class DownloadResult: - suite: str - rows: int - path: str - source: str - note: str = "" - - -def fetch_bytes(url: str, timeout: float, headers: dict[str, str] | None = None) -> bytes: - request = urllib.request.Request(url, headers=headers or {}) - with urllib.request.urlopen(request, timeout=timeout) as response: - return response.read() - - -def fetch_json( - endpoint: str, - params: dict[str, object], - headers: dict[str, str] | None, - timeout: float, -) -> object: - query = urllib.parse.urlencode(params, doseq=True) - url = f"{HF_DATASET_VIEWER_BASE}/{endpoint}" - if query: - url = f"{url}?{query}" - raw = fetch_bytes(url, timeout=timeout, headers=headers) - return json.loads(raw.decode("utf-8")) - - -def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> int: - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8") as handle: - for row in rows: - handle.write(json.dumps(row, ensure_ascii=False) + "\n") - return len(rows) - - -# --------------------------------------------------------------------------- -# HuggingFace `datasets` library helpers -# --------------------------------------------------------------------------- - -def _load_hf_dataset( - dataset_name: str, - config: str | None = None, - split: str = "test", -) -> list[dict[str, Any]]: - """Load a dataset using the HuggingFace `datasets` library.""" - from datasets import load_dataset # type: ignore[import-untyped] - - kwargs: dict[str, Any] = {} - if config: - kwargs["name"] = config - - ds = load_dataset(dataset_name, split=split, **kwargs) - return [dict(row) for row in ds] # type: ignore[union-attr] - - -def _try_load_hf( - dataset_name: str, - config: str | None = None, - split_preference: tuple[str, ...] = ("test", "validation", "train"), -) -> list[dict[str, Any]]: - """Try loading with preferred splits, falling back through the list.""" - for split in split_preference: - try: - rows = _load_hf_dataset(dataset_name, config=config, split=split) - if rows: - print(f" Loaded {len(rows)} rows from {dataset_name} [{split}]") - return rows - except (ValueError, KeyError): - continue - raise ValueError(f"No valid split found for {dataset_name}") - - -def _fetch_hf_rows( - dataset: str, - *, - config_preference: tuple[str, ...] = (), - split_preference: tuple[str, ...] = ("test", "validation", "train"), - json_fetcher: JsonFetcher = fetch_json, - timeout: float = 60.0, - headers: dict[str, str] | None = None, -) -> list[dict[str, Any]]: - """Legacy REST-API fetcher (kept for backward compatibility with tests).""" - splits_payload = json_fetcher("splits", {"dataset": dataset}, headers, timeout) - splits = list((splits_payload or {}).get("splits", [])) # type: ignore[assignment] - if not splits: - return [] - - chosen: dict[str, Any] | None = None - for config_name in config_preference: - for split_name in split_preference: - chosen = next( - ( - item for item in splits - if item.get("config") == config_name and item.get("split") == split_name - ), - None, - ) - if chosen is not None: - break - if chosen is not None: - break - if chosen is None: - for split_name in split_preference: - chosen = next((item for item in splits if item.get("split") == split_name), None) - if chosen is not None: - break - if chosen is None: - chosen = splits[0] - - rows: list[dict[str, Any]] = [] - offset = 0 - length = 100 - while True: - payload = json_fetcher( - "rows", - { - "dataset": chosen["dataset"], - "config": chosen["config"], - "split": chosen["split"], - "offset": offset, - "length": length, - }, - headers, - timeout, - ) - batch = [item.get("row", {}) for item in (payload or {}).get("rows", [])] # type: ignore[union-attr] - rows.extend(batch) - total = int((payload or {}).get("num_rows_total", len(rows))) # type: ignore[union-attr] - offset += len(batch) - if not batch or offset >= total: - break - return rows - - -# --------------------------------------------------------------------------- -# Answer extraction helpers -# --------------------------------------------------------------------------- - -def _extract_gsm8k_answer(text: str) -> str: - if "####" in text: - text = text.split("####", 1)[1] - numbers = re.findall(r"-?\d[\d,]*\.?\d*", text.replace("$", "")) - if numbers: - return numbers[-1].replace(",", "") - return text.strip().replace(",", "") - - -def _extract_math_answer(solution: str) -> str: - boxed_fraction = re.search(r"\\boxed\{\\frac\{([^}]+)\}\{([^}]+)\}\}", solution, flags=re.DOTALL) - if boxed_fraction: - return f"{boxed_fraction.group(1).strip()}/{boxed_fraction.group(2).strip()}" - boxed = re.search(r"\\boxed\{([^{}]+)\}", solution, flags=re.DOTALL) - value = boxed.group(1) if boxed else solution - value = value.strip() - value = value.replace("\\frac{", "").replace("}{", "/").replace("}", "") - value = value.replace("$", "").replace(",", "").strip() - fraction = re.search(r"-?\d+\s*/\s*-?\d+", value) - if fraction: - return fraction.group(0).replace(" ", "") - numbers = re.findall(r"-?\d+(?:/\d+)?(?:\.\d+)?", value) - return numbers[-1] if numbers else value - - -# --------------------------------------------------------------------------- -# Individual dataset downloaders (using `datasets` library) -# --------------------------------------------------------------------------- - -def _download_humaneval(output_path: Path, *, timeout: float) -> DownloadResult: - raw = fetch_bytes(HUMANEVAL_GZ_URL, timeout=timeout) - if raw[:2] == b"\x1f\x8b": - raw = gzip.decompress(raw) - lines = [json.loads(line) for line in raw.decode("utf-8").splitlines() if line.strip()] - rows = [ - { - "task_id": item["task_id"], - "prompt": item["prompt"], - "canonical_solution": item.get("canonical_solution", ""), - "test": item["test"], - "entry_point": item["entry_point"], - } - for item in lines - ] - count = _write_jsonl(output_path, rows) - return DownloadResult("humaneval", count, str(output_path), "official") - - -def _download_gsm8k( - output_path: Path, - *, - timeout: float, - json_fetcher: JsonFetcher | None = None, -) -> DownloadResult: - if json_fetcher is not None: - # Legacy path for tests - rows = _fetch_hf_rows( - "openai/gsm8k", - config_preference=("main",), - split_preference=("test",), - json_fetcher=json_fetcher, - timeout=timeout, - ) - else: - rows = _try_load_hf("openai/gsm8k", config="main", split_preference=("test",)) - normalized = [ - { - "id": f"gsm8k-{index + 1:04d}", - "question": row["question"], - "answer": _extract_gsm8k_answer(str(row["answer"])), - } - for index, row in enumerate(rows) - ] - count = _write_jsonl(output_path, normalized) - return DownloadResult("gsm8k", count, str(output_path), "official") - - -def _download_mbpp(output_path: Path, *, timeout: float) -> DownloadResult: - try: - rows = _try_load_hf("google-research-datasets/mbpp", config="sanitized", split_preference=("test", "validation")) - except Exception: - rows = _try_load_hf("google-research-datasets/mbpp", config="full", split_preference=("test", "validation")) - normalized = [ - { - "task_id": row.get("task_id", index + 1), - "text": row.get("text") or row.get("prompt") or "", - "code": row.get("code", ""), - "test_list": row.get("test_list") or row.get("test_setup_code", []), - } - for index, row in enumerate(rows) - ] - count = _write_jsonl(output_path, normalized) - return DownloadResult("mbpp", count, str(output_path), "official") - - -def _download_math(output_path: Path, *, timeout: float) -> DownloadResult: - rows = _try_load_hf("DigitalLearningGmbH/MATH-lighteval", split_preference=("test", "train")) - normalized = [ - { - "id": row.get("problem_id", f"math-{index + 1:04d}"), - "problem": row.get("problem", ""), - "answer": _extract_math_answer(str(row.get("solution", row.get("answer", "")))), - "subject": row.get("type", row.get("subject", "unknown")), - "level": row.get("level", 0), - } - for index, row in enumerate(rows) - ] - count = _write_jsonl(output_path, normalized) - return DownloadResult("math", count, str(output_path), "official") - - -def _download_mmlu_pro(output_path: Path, *, timeout: float) -> DownloadResult: - rows = _try_load_hf("TIGER-Lab/MMLU-Pro", split_preference=("test", "validation")) - letters = "ABCDEFGHIJ" - normalized = [] - for index, row in enumerate(rows): - answer_raw = row.get("answer", "") - if isinstance(answer_raw, int) and answer_raw < len(letters): - answer = letters[answer_raw] - else: - answer = str(answer_raw) - normalized.append({ - "id": f"mmlu-pro-{index + 1:04d}", - "subject": row.get("category", row.get("subject", "unknown")), - "question": row.get("question", ""), - "choices": row.get("options", row.get("choices", [])), - "answer": answer, - }) - count = _write_jsonl(output_path, normalized) - return DownloadResult("mmlu-pro", count, str(output_path), "official") - - -def _download_gpqa(output_path: Path, *, timeout: float) -> DownloadResult: - rows = _try_load_hf("Idavidrein/gpqa", config="gpqa_diamond", split_preference=("train",)) - normalized = [] - for index, row in enumerate(rows): - choices = [ - row.get("Correct Answer", ""), - row.get("Incorrect Answer 1", ""), - row.get("Incorrect Answer 2", ""), - row.get("Incorrect Answer 3", ""), - ] - normalized.append({ - "id": f"gpqa-{index + 1:04d}", - "subject": row.get("Subdomain", row.get("domain", "science")), - "question": row.get("Question", ""), - "choices": choices, - "answer": "A", - }) - count = _write_jsonl(output_path, normalized) - return DownloadResult("gpqa-diamond", count, str(output_path), "official") - - -def _download_bigbench_hard(output_path: Path, *, timeout: float) -> DownloadResult: - from datasets import load_dataset # type: ignore[import-untyped] - - configs = [ - "boolean_expressions", "causal_judgement", "date_understanding", - "disambiguation_qa", "dyck_languages", "formal_fallacies", - "geometric_shapes", "hyperbaton", "logical_deduction_three_objects", - "logical_deduction_five_objects", "logical_deduction_seven_objects", - "movie_recommendation", "multistep_arithmetic_two", "navigate", - "object_counting", "penguins_in_a_table", - "reasoning_about_colored_objects", "ruin_names", - "salient_translation_error_detection", "snarks", - "sports_understanding", "temporal_sequences", - "tracking_shuffled_objects_three_objects", - "tracking_shuffled_objects_five_objects", - "tracking_shuffled_objects_seven_objects", - "web_of_lies", "word_sorting", - ] - all_rows: list[dict[str, Any]] = [] - for config in configs: - try: - ds = load_dataset("lukaemon/bbh", config, split="test") - for row in ds: - row_dict = dict(row) # type: ignore[arg-type] - row_dict["task"] = config - all_rows.append(row_dict) - except Exception: - continue - print(f" Loaded {len(all_rows)} rows from lukaemon/bbh [{len(configs)} tasks]") - normalized = [] - for index, row in enumerate(all_rows): - target = row.get("target", row.get("answer", "")) - normalized.append({ - "id": f"bbh-{index + 1:05d}", - "task": row.get("task", "unknown"), - "question": row.get("input", row.get("question", "")), - "choices": [], - "answer": str(target).strip(), - }) - count = _write_jsonl(output_path, normalized) - return DownloadResult("bigbench-hard", count, str(output_path), "official") - - -def _download_mmmlu(output_path: Path, *, timeout: float) -> DownloadResult: - rows = _try_load_hf("openai/MMMLU", split_preference=("test", "validation")) - letters = "ABCD" - normalized = [] - for index, row in enumerate(rows): - answer_raw = row.get("answer", "") - if isinstance(answer_raw, int) and answer_raw < len(letters): - answer = letters[answer_raw] - else: - answer = str(answer_raw) - normalized.append({ - "id": f"mmmlu-{index + 1:04d}", - "language": row.get("language", "en"), - "subject": row.get("subject", "unknown"), - "question": row.get("question", ""), - "choices": row.get("choices", row.get("options", [])), - "answer": answer, - }) - count = _write_jsonl(output_path, normalized) - return DownloadResult("mmmlu", count, str(output_path), "official") - - -def _download_hle(output_path: Path, *, timeout: float) -> DownloadResult: - rows = _try_load_hf("cais/hle", split_preference=("test", "validation", "train")) - normalized = [] - for index, row in enumerate(rows): - entry: dict[str, Any] = { - "id": f"hle-{index + 1:04d}", - "subject": row.get("category", row.get("subject", "general")), - "question": row.get("question", ""), - "answer": str(row.get("answer", "")), - } - if row.get("choices") or row.get("options"): - entry["answer_type"] = "multiple_choice" - entry["choices"] = row.get("choices", row.get("options", [])) - else: - entry["answer_type"] = "exact" - normalized.append(entry) - count = _write_jsonl(output_path, normalized) - return DownloadResult("hle", count, str(output_path), "official") - - -def _export_builtin(output_path: Path, suite: str, rows: list[dict[str, Any]], *, source: str = "builtin", note: str = "") -> DownloadResult: - count = _write_jsonl(output_path, rows) - return DownloadResult(suite, count, str(output_path), source, note) - - -def _builtin_rows(suite: str) -> list[dict[str, Any]]: - mapping: dict[str, list[dict[str, Any]]] = { - "humaneval": list(_HUMANEVAL_BUILTINS), - "mbpp": list(_MBPP_BUILTINS), - "gsm8k": list(_GSM8K_BUILTINS), - "math": list(_MATH_BUILTINS), - "swe-bench": list(_SWE_BUILTINS), - "aider": list(_AIDER_BUILTINS), - "livecodebench": list(_LIVECODEBENCH_BUILTINS), - "aime": list(_AIME_BUILTINS), - "ifeval": list(_IFEVAL_BUILTINS), - "bfcl": list(_BFCL_BUILTINS), - "mmlu-pro": list(_MMLU_PRO_BUILTINS), - "gpqa-diamond": list(_GPQA_BUILTINS), - "bigbench-hard": list(_BIGBENCH_BUILTINS), - "mmmlu": list(_MMMLU_BUILTINS), - "hle": list(_HLE_BUILTINS), - "tau2": list(_TAU2_BUILTINS), - "codeforces": list(_CODEFORCES_BUILTINS), - } - return list(mapping[suite]) - - -def prepare_suite( - suite: str, - *, - data_dir: Path, - force: bool, - builtin_only: bool, - official_only: bool, - timeout: float, -) -> DownloadResult: - output_map = { - "mmlu-pro": "mmlu_pro.jsonl", - "gpqa-diamond": "gpqa.jsonl", - "bigbench-hard": "bigbench_hard.jsonl", - "mmmlu": "mmmlu.jsonl", - "hle": "hle.jsonl", - "tau2": "tau2.jsonl", - "codeforces": "codeforces.jsonl", - } - output_path = data_dir / output_map.get(suite, f"{suite}.jsonl") - if output_path.exists() and not force: - lines = [line for line in output_path.read_text(encoding="utf-8").splitlines() if line.strip()] - return DownloadResult(suite, len(lines), str(output_path), "existing") - - data_dir.mkdir(parents=True, exist_ok=True) - - builtin_only_suites = {"swe-bench", "aider", "livecodebench", "aime", "ifeval", "bfcl", "tau2", "codeforces"} - official_downloaders = { - "humaneval": _download_humaneval, - "gsm8k": _download_gsm8k, - "mbpp": _download_mbpp, - "math": _download_math, - "mmlu-pro": _download_mmlu_pro, - "gpqa-diamond": _download_gpqa, - "bigbench-hard": _download_bigbench_hard, - "mmmlu": _download_mmmlu, - "hle": _download_hle, - } - - if suite in builtin_only_suites or builtin_only: - return _export_builtin(output_path, suite, _builtin_rows(suite)) - - downloader = official_downloaders.get(suite) - if downloader is None: - return _export_builtin(output_path, suite, _builtin_rows(suite)) - - try: - return downloader(output_path, timeout=timeout) - except Exception as exc: - if official_only: - raise - note = f"official download failed: {exc}" - print(f" WARNING: {note}") - print(f" Falling back to {len(_builtin_rows(suite))} built-in problems.") - print(f" To get full data, install `datasets`: pip install datasets") - return _export_builtin( - output_path, - suite, - _builtin_rows(suite), - source="builtin-fallback", - note=note, - ) - - -def _write_manifest(data_dir: Path, results: list[DownloadResult]) -> Path: - manifest_path = data_dir / "manifest.json" - payload = { - "generated_at": __import__("time").strftime("%Y-%m-%dT%H:%M:%S"), - "results": [asdict(item) for item in results], - } - manifest_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - return manifest_path - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Download benchmark datasets for claw-code-agent.") - parser.add_argument("--suite", action="append", default=[], help="Suite to prepare. Can be repeated.") - parser.add_argument("--all", action="store_true", help="Prepare all known suites.") - parser.add_argument("--list", action="store_true", help="List known suites.") - parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR, help="Output data directory.") - parser.add_argument("--force", action="store_true", help="Overwrite existing files.") - parser.add_argument("--builtin-only", action="store_true", help="Skip official downloads and export builtins only.") - parser.add_argument("--official-only", action="store_true", help="Do not fall back to builtins.") - parser.add_argument("--timeout", type=float, default=60.0, help="Network timeout in seconds.") - return parser - - -def main() -> None: - parser = build_parser() - args = parser.parse_args() - known = [ - "humaneval", - "mbpp", - "gsm8k", - "math", - "swe-bench", - "aider", - "livecodebench", - "aime", - "ifeval", - "bfcl", - "mmlu-pro", - "gpqa-diamond", - "bigbench-hard", - "mmmlu", - "hle", - "tau2", - "codeforces", - ] - - if args.list: - for name in known: - print(name) - return - - suites = list(args.suite) - if args.all: - suites = known - if not suites: - parser.error("specify --suite or --all") - - results = [ - prepare_suite( - suite, - data_dir=args.data_dir, - force=args.force, - builtin_only=args.builtin_only, - official_only=args.official_only, - timeout=args.timeout, - ) - for suite in suites - ] - manifest = _write_manifest(args.data_dir, results) - print(f"Wrote {len(results)} suite files to {args.data_dir}") - print(f"Manifest: {manifest}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/run.py b/benchmarks/run.py deleted file mode 100644 index c478601..0000000 --- a/benchmarks/run.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python3 -""" -Local benchmark runner for claw-code-agent. - -Runs the REAL agent binary against a suite of coding tasks and scores -pass/fail automatically. No Docker required. - -Usage: - # Run all tasks - python3 -m benchmarks.run - - # Run a single task - python3 -m benchmarks.run --task file-create-basic - - # Run a category - python3 -m benchmarks.run --category bugfix - - # Run a difficulty level - python3 -m benchmarks.run --difficulty easy - - # List available tasks - python3 -m benchmarks.run --list - - # Verbose output - python3 -m benchmarks.run --verbose -""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import subprocess -import sys -import tempfile -import time -from dataclasses import dataclass -from pathlib import Path - -from benchmarks.tasks.definitions import TASKS, BenchmarkTask, get_task -from benchmarks.suites.base import make_temp_workspace - - -# --------------------------------------------------------------------------- -# Result tracking -# --------------------------------------------------------------------------- - -@dataclass -class TaskResult: - task_id: str - category: str - difficulty: str - passed: bool - duration_sec: float - agent_exit_code: int - verify_exit_code: int - error: str = "" - - -# --------------------------------------------------------------------------- -# Task execution -# --------------------------------------------------------------------------- - -def _run_shell(cmd: str, cwd: str, timeout: float = 30.0) -> tuple[int, str]: - """Run a shell command, return (exit_code, combined_output).""" - try: - proc = subprocess.run( - cmd, - shell=True, - cwd=cwd, - capture_output=True, - text=True, - timeout=timeout, - ) - output = (proc.stdout + proc.stderr).strip() - return proc.returncode, output - except subprocess.TimeoutExpired: - return 1, f"[TIMEOUT after {timeout}s]" - except Exception as exc: - return 1, str(exc) - - -def run_task( - task: BenchmarkTask, - *, - project_root: str, - agent_timeout: float = 300.0, - verbose: bool = False, -) -> TaskResult: - """Run a single benchmark task end-to-end.""" - - # Create isolated temp workspace - workspace = make_temp_workspace("claw_bench", task.category, task.id) - - if verbose: - print(f" workspace: {workspace}") - - try: - # --- SETUP --- - if task.setup: - code, out = _run_shell(task.setup, cwd=workspace) - if code != 0: - return TaskResult( - task_id=task.id, - category=task.category, - difficulty=task.difficulty, - passed=False, - duration_sec=0.0, - agent_exit_code=-1, - verify_exit_code=-1, - error=f"Setup failed: {out}", - ) - - # --- RUN AGENT --- - agent_cmd = ( - f"{sys.executable} -m src.main agent " - f"{_shell_quote(task.instruction)} " - f"--cwd {_shell_quote(workspace)} " - f"--allow-write " - f"--allow-shell" - ) - - if verbose: - print(f" agent cmd: {agent_cmd[:120]}...") - - start = time.time() - agent_code, agent_out = _run_shell( - agent_cmd, - cwd=project_root, - timeout=agent_timeout, - ) - duration = time.time() - start - - if verbose: - print(f" agent exit={agent_code} duration={duration:.1f}s") - if agent_out: - # Print last few lines of agent output - lines = agent_out.split("\n") - for line in lines[-5:]: - print(f" > {line}") - - # --- VERIFY --- - verify_code, verify_out = _run_shell(task.verify, cwd=workspace, timeout=30.0) - - if verbose: - status = "PASS" if verify_code == 0 else "FAIL" - print(f" verify exit={verify_code} -> {status}") - if verify_code != 0 and verify_out: - print(f" verify output: {verify_out[:200]}") - - return TaskResult( - task_id=task.id, - category=task.category, - difficulty=task.difficulty, - passed=(verify_code == 0), - duration_sec=duration, - agent_exit_code=agent_code, - verify_exit_code=verify_code, - error=verify_out if verify_code != 0 else "", - ) - - finally: - # Clean up workspace - shutil.rmtree(workspace, ignore_errors=True) - - -def _shell_quote(s: str) -> str: - """Quote a string for shell use.""" - import shlex - return shlex.quote(s) - - -# --------------------------------------------------------------------------- -# Reporting -# --------------------------------------------------------------------------- - -def print_results(results: list[TaskResult]) -> None: - """Print a formatted results table.""" - - total = len(results) - passed = sum(1 for r in results if r.passed) - failed = total - passed - - print() - print("=" * 72) - print(" CLAW CODE AGENT — BENCHMARK RESULTS") - print("=" * 72) - print() - print(f" {'Task ID':<30} {'Category':<12} {'Diff':<8} {'Result':<8} {'Time':>6}") - print(f" {'─' * 30} {'─' * 12} {'─' * 8} {'─' * 8} {'─' * 6}") - - for r in results: - status = "PASS" if r.passed else "FAIL" - icon = " ✅" if r.passed else " ❌" - time_str = f"{r.duration_sec:.1f}s" - print(f"{icon} {r.task_id:<30} {r.category:<12} {r.difficulty:<8} {status:<8} {time_str:>6}") - - print() - print("─" * 72) - print(f" Total: {total} | Passed: {passed} | Failed: {failed} | Score: {passed}/{total} ({100*passed/total:.0f}%)") - print("─" * 72) - - # Breakdown by category - categories: dict[str, list[TaskResult]] = {} - for r in results: - categories.setdefault(r.category, []).append(r) - - print() - print(" Category Breakdown:") - for cat, cat_results in sorted(categories.items()): - cat_passed = sum(1 for r in cat_results if r.passed) - cat_total = len(cat_results) - bar = "█" * cat_passed + "░" * (cat_total - cat_passed) - print(f" {cat:<14} {bar} {cat_passed}/{cat_total}") - - # Breakdown by difficulty - difficulties: dict[str, list[TaskResult]] = {} - for r in results: - difficulties.setdefault(r.difficulty, []).append(r) - - print() - print(" Difficulty Breakdown:") - for diff in ("easy", "medium", "hard"): - if diff in difficulties: - diff_results = difficulties[diff] - diff_passed = sum(1 for r in diff_results if r.passed) - diff_total = len(diff_results) - print(f" {diff:<14} {diff_passed}/{diff_total} ({100*diff_passed/diff_total:.0f}%)") - - total_time = sum(r.duration_sec for r in results) - print() - print(f" Total time: {total_time:.1f}s") - print() - - -def save_results(results: list[TaskResult], output_path: str) -> None: - """Save results to JSON.""" - data = { - "benchmark": "claw-code-agent-local", - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), - "model": os.environ.get("OPENAI_MODEL", "unknown"), - "total": len(results), - "passed": sum(1 for r in results if r.passed), - "score_pct": round(100 * sum(1 for r in results if r.passed) / len(results), 1) if results else 0, - "results": [ - { - "task_id": r.task_id, - "category": r.category, - "difficulty": r.difficulty, - "passed": r.passed, - "duration_sec": round(r.duration_sec, 2), - "agent_exit_code": r.agent_exit_code, - "verify_exit_code": r.verify_exit_code, - "error": r.error, - } - for r in results - ], - } - with open(output_path, "w") as f: - json.dump(data, f, indent=2) - print(f" Results saved to {output_path}") - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Claw Code Agent local benchmark") - parser.add_argument("--task", help="Run a single task by ID") - parser.add_argument("--category", help="Run tasks in a category") - parser.add_argument("--difficulty", choices=["easy", "medium", "hard"], help="Run tasks by difficulty") - parser.add_argument("--list", action="store_true", help="List available tasks") - parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") - parser.add_argument("--timeout", type=float, default=300.0, help="Agent timeout per task in seconds (default: 300)") - parser.add_argument("--output", "-o", help="Save results to JSON file") - args = parser.parse_args() - - if args.list: - print(f"\n {'ID':<30} {'Category':<12} {'Difficulty':<10}") - print(f" {'─' * 30} {'─' * 12} {'─' * 10}") - for t in TASKS: - print(f" {t.id:<30} {t.category:<12} {t.difficulty:<10}") - print(f"\n Total: {len(TASKS)} tasks\n") - return - - # Select tasks - tasks_to_run: list[BenchmarkTask] = [] - - if args.task: - t = get_task(args.task) - if t is None: - print(f"Unknown task: {args.task}") - print("Use --list to see available tasks") - sys.exit(1) - tasks_to_run = [t] - else: - tasks_to_run = list(TASKS) - if args.category: - tasks_to_run = [t for t in tasks_to_run if t.category == args.category] - if args.difficulty: - tasks_to_run = [t for t in tasks_to_run if t.difficulty == args.difficulty] - - if not tasks_to_run: - print("No tasks matched the filters.") - sys.exit(1) - - # Find project root - project_root = str(Path(__file__).resolve().parent.parent) - - # Check environment - model = os.environ.get("OPENAI_MODEL", "not set") - base_url = os.environ.get("OPENAI_BASE_URL", "not set") - - print() - print("=" * 72) - print(" CLAW CODE AGENT — LOCAL BENCHMARK") - print("=" * 72) - print(f" Model: {model}") - print(f" Base URL: {base_url}") - print(f" Tasks: {len(tasks_to_run)}") - print(f" Timeout: {args.timeout}s per task") - print("=" * 72) - print() - - # Run tasks - results: list[TaskResult] = [] - - for i, task in enumerate(tasks_to_run, 1): - print(f"[{i}/{len(tasks_to_run)}] {task.id} ({task.category}, {task.difficulty})") - - result = run_task( - task, - project_root=project_root, - agent_timeout=args.timeout, - verbose=args.verbose, - ) - results.append(result) - - status = "PASS ✅" if result.passed else "FAIL ❌" - print(f" -> {status} ({result.duration_sec:.1f}s)") - print() - - # Report - print_results(results) - - if args.output: - save_results(results, args.output) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/run_suite.py b/benchmarks/run_suite.py deleted file mode 100644 index 86f4757..0000000 --- a/benchmarks/run_suite.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env python3 -""" -Standard evaluation benchmark runner for claw-code-agent. - -Runs the agent against well-known evaluation suites and reports scores. - -Usage: - # List available benchmark suites - python3 -m benchmarks.run_suite --list - - # Run a specific suite (built-in subset) - python3 -m benchmarks.run_suite --suite humaneval - python3 -m benchmarks.run_suite --suite mbpp - python3 -m benchmarks.run_suite --suite swe-bench - python3 -m benchmarks.run_suite --suite aider - python3 -m benchmarks.run_suite --suite livecodebench - python3 -m benchmarks.run_suite --suite math - python3 -m benchmarks.run_suite --suite gsm8k - python3 -m benchmarks.run_suite --suite aime - python3 -m benchmarks.run_suite --suite ifeval - python3 -m benchmarks.run_suite --suite bfcl - - # Run ALL suites - python3 -m benchmarks.run_suite --all - - # Run by category - python3 -m benchmarks.run_suite --category coding - python3 -m benchmarks.run_suite --category math - python3 -m benchmarks.run_suite --category instruction-following - - # Limit problems per suite (for quick testing) - python3 -m benchmarks.run_suite --suite humaneval --limit 5 - - # Verbose output + save results - python3 -m benchmarks.run_suite --suite humaneval -v -o results.json -""" - -from __future__ import annotations - -import argparse -import json -import sys -import time - -from benchmarks.suites.base import BenchmarkSuite, SuiteReport - -# Import all suites -from benchmarks.suites.humaneval import HumanEvalBenchmark -from benchmarks.suites.mbpp import MBPPBenchmark -from benchmarks.suites.swe_bench import SWEBenchBenchmark -from benchmarks.suites.aider import AiderBenchmark -from benchmarks.suites.livecodebench import LiveCodeBenchBenchmark -from benchmarks.suites.math_bench import MATHBenchmark -from benchmarks.suites.gsm8k import GSM8KBenchmark -from benchmarks.suites.aime import AIMEBenchmark -from benchmarks.suites.ifeval import IFEvalBenchmark -from benchmarks.suites.bfcl import BFCLBenchmark -from benchmarks.suites.mmlu_pro import MMLUProBenchmark -from benchmarks.suites.gpqa import GPQABenchmark -from benchmarks.suites.bigbench import BigBenchHardBenchmark -from benchmarks.suites.mmmlu import MMMMLUBenchmark -from benchmarks.suites.hle import HLEBenchmark -from benchmarks.suites.tau2 import Tau2Benchmark -from benchmarks.suites.codeforces import CodeforcesBenchmark - - -# --------------------------------------------------------------------------- -# Registry -# --------------------------------------------------------------------------- - -SUITE_REGISTRY: dict[str, type[BenchmarkSuite]] = { - "humaneval": HumanEvalBenchmark, - "mbpp": MBPPBenchmark, - "swe-bench": SWEBenchBenchmark, - "aider": AiderBenchmark, - "livecodebench": LiveCodeBenchBenchmark, - "math": MATHBenchmark, - "gsm8k": GSM8KBenchmark, - "aime": AIMEBenchmark, - "ifeval": IFEvalBenchmark, - "bfcl": BFCLBenchmark, - "mmlu-pro": MMLUProBenchmark, - "gpqa-diamond": GPQABenchmark, - "bigbench-hard": BigBenchHardBenchmark, - "mmmlu": MMMMLUBenchmark, - "hle": HLEBenchmark, - "tau2": Tau2Benchmark, - "codeforces": CodeforcesBenchmark, -} - -CATEGORY_MAP: dict[str, list[str]] = { - "coding": ["humaneval", "mbpp", "swe-bench", "aider", "livecodebench", "codeforces"], - "math": ["math", "gsm8k", "aime"], - "instruction-following": ["ifeval", "bfcl"], - "knowledge": ["mmlu-pro", "gpqa-diamond", "mmmlu", "hle"], - "reasoning": ["bigbench-hard", "tau2"], -} - - -# --------------------------------------------------------------------------- -# Multi-suite reporting -# --------------------------------------------------------------------------- - -def print_combined_report(reports: list[SuiteReport]) -> None: - """Print a combined summary of all suite runs.""" - print() - print("=" * 80) - print(" COMBINED BENCHMARK REPORT") - print("=" * 80) - print() - print(f" {'Suite':<20} {'Category':<22} {'Passed':>8} {'Total':>8} {'Score':>8} {'Time':>8}") - print(f" {'─' * 20} {'─' * 22} {'─' * 8} {'─' * 8} {'─' * 8} {'─' * 8}") - - total_passed = 0 - total_problems = 0 - total_time = 0.0 - - for r in reports: - suite_cls = SUITE_REGISTRY.get(r.suite_name.lower().replace("-", "").replace("_", "")) - cat = suite_cls.category if suite_cls else "unknown" - print( - f" {r.suite_name:<20} {cat:<22} " - f"{r.passed:>8} {r.total:>8} " - f"{r.score_pct:>7.1f}% {r.duration_sec:>7.1f}s" - ) - total_passed += r.passed - total_problems += r.total - total_time += r.duration_sec - - print() - print("─" * 80) - overall_pct = round(100.0 * total_passed / total_problems, 1) if total_problems else 0.0 - print( - f" OVERALL: {total_passed}/{total_problems} ({overall_pct}%) " - f"in {total_time:.1f}s" - ) - print("─" * 80) - - # Category breakdown - print() - print(" Category Breakdown:") - for cat_name, suite_names in CATEGORY_MAP.items(): - cat_reports = [r for r in reports if r.suite_name.lower().replace("-", "").replace("_", "") in - [s.replace("-", "").replace("_", "") for s in suite_names]] - if cat_reports: - cp = sum(r.passed for r in cat_reports) - ct = sum(r.total for r in cat_reports) - cpct = round(100.0 * cp / ct, 1) if ct else 0.0 - bar_len = 20 - filled = round(bar_len * cp / ct) if ct else 0 - bar = "█" * filled + "░" * (bar_len - filled) - print(f" {cat_name:<22} {bar} {cp}/{ct} ({cpct}%)") - - print() - - -def save_combined_report(reports: list[SuiteReport], path: str) -> None: - data = { - "benchmark_run": "claw-code-agent-evaluation", - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), - "suites": [r.to_dict() for r in reports], - "summary": { - "total_suites": len(reports), - "total_problems": sum(r.total for r in reports), - "total_passed": sum(r.passed for r in reports), - "overall_score_pct": round( - 100.0 * sum(r.passed for r in reports) / sum(r.total for r in reports), 1 - ) if any(r.total for r in reports) else 0.0, - }, - } - with open(path, "w") as fh: - json.dump(data, fh, indent=2) - print(f" Combined results saved to {path}") - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser( - description="Run standard evaluation benchmarks against claw-code-agent", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=( - "Examples:\n" - " python3 -m benchmarks.run_suite --list\n" - " python3 -m benchmarks.run_suite --suite humaneval\n" - " python3 -m benchmarks.run_suite --suite humaneval --limit 5 -v\n" - " python3 -m benchmarks.run_suite --category coding\n" - " python3 -m benchmarks.run_suite --all\n" - " python3 -m benchmarks.run_suite --all -o results.json\n" - ), - ) - parser.add_argument("--suite", action="append", default=[], - help="Run a specific benchmark suite (can be repeated)") - parser.add_argument("--category", choices=list(CATEGORY_MAP.keys()), - help="Run all suites in a category") - parser.add_argument("--all", action="store_true", - help="Run ALL benchmark suites") - parser.add_argument("--list", action="store_true", - help="List available benchmark suites") - parser.add_argument("--limit", type=int, default=None, - help="Max problems per suite (for quick testing)") - parser.add_argument("--timeout", type=float, default=300.0, - help="Agent timeout per problem in seconds (default: 300)") - parser.add_argument("--verbose", "-v", action="store_true", - help="Verbose output") - parser.add_argument("--output", "-o", - help="Save results to JSON file") - parser.add_argument("--data-dir", - help="Directory containing dataset files (JSONL)") - parser.add_argument("--artifacts-dir", - help="Directory where per-problem artifacts will be saved") - parser.add_argument("--save-passing-artifacts", action="store_true", - help="Also save artifacts for passing problems") - args = parser.parse_args() - - if args.list: - print() - print(" Available Benchmark Suites:") - print(" " + "─" * 68) - print(f" {'Name':<18} {'Category':<22} {'Description'}") - print(" " + "─" * 68) - for name, cls in SUITE_REGISTRY.items(): - print(f" {name:<18} {cls.category:<22} {cls.description}") - print() - print(" Categories:") - for cat, suites in CATEGORY_MAP.items(): - print(f" {cat}: {', '.join(suites)}") - print() - return - - # Determine which suites to run - suite_names: list[str] = [] - if args.all: - suite_names = list(SUITE_REGISTRY.keys()) - elif args.category: - suite_names = CATEGORY_MAP.get(args.category, []) - elif args.suite: - for s in args.suite: - if s not in SUITE_REGISTRY: - print(f"Error: unknown suite '{s}'. Available: {', '.join(SUITE_REGISTRY)}") - sys.exit(1) - suite_names = list(args.suite) - else: - parser.print_help() - print("\nError: specify --suite, --category, or --all") - sys.exit(1) - - if not suite_names: - print("No suites matched.") - sys.exit(1) - - # Run suites - reports: list[SuiteReport] = [] - for name in suite_names: - cls = SUITE_REGISTRY[name] - suite = cls( - data_dir=args.data_dir, - limit=args.limit, - agent_timeout=args.timeout, - verbose=args.verbose, - artifacts_dir=args.artifacts_dir, - save_passing_artifacts=args.save_passing_artifacts, - ) - report = suite.run_all() - reports.append(report) - - # Combined report - if len(reports) > 1: - print_combined_report(reports) - - if args.output: - save_combined_report(reports, args.output) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/run_terminal_bench_local.py b/benchmarks/run_terminal_bench_local.py deleted file mode 100644 index beea990..0000000 --- a/benchmarks/run_terminal_bench_local.py +++ /dev/null @@ -1,541 +0,0 @@ -#!/usr/bin/env python3 -""" -Run Terminal-Bench tasks locally with Apptainer and claw-code-agent. - -This runner is designed for cluster setups where Docker is unavailable but -Apptainer is available and the model server runs on the same node. -""" - -from __future__ import annotations - -import argparse -import fnmatch -import json -import os -import re -import shlex -import shutil -import subprocess -import sys -import time -import tomllib -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Any - - -DEFAULT_TASKS_DIR = Path.home() / ".cache/harbor/tasks/packages/terminal-bench" -DEFAULT_JOBS_DIR = Path("jobs/terminal_bench_local") - - -@dataclass -class TerminalBenchTask: - task_dir: Path - name: str - short_name: str - instruction: str - docker_image: str | None - agent_timeout_sec: float | None - verifier_timeout_sec: float - workdir: str - has_docker_compose: bool - raw_config: dict[str, Any] = field(default_factory=dict) - - -@dataclass -class LocalTrialResult: - task_name: str - short_name: str - passed: bool - reward: float | None - duration_sec: float - status: str - image: str | None - workdir: str - trial_dir: str - error: str = "" - agent_return_code: int | None = None - verifier_return_code: int | None = None - metadata: dict[str, Any] = field(default_factory=dict) - - -def _load_toml(path: Path) -> dict[str, Any]: - return tomllib.loads(path.read_text(encoding="utf-8")) - - -def strip_canary(text: str) -> str: - lines = text.splitlines() - index = 0 - while index < len(lines): - stripped = lines[index].strip() - if stripped.startswith("