update the codebase and clean up it

benchmarks/download_datasets.py

@@ -0,0 +1,391 @@
+#!/usr/bin/env python3
+"""
+Download or export benchmark datasets into benchmarks/data.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import json
+import re
+import urllib.parse
+import urllib.request
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from benchmarks.suites.aider import _BUILTIN_PROBLEMS as _AIDER_BUILTINS
+from benchmarks.suites.aime import _BUILTIN_PROBLEMS as _AIME_BUILTINS
+from benchmarks.suites.bfcl import _BUILTIN_PROBLEMS as _BFCL_BUILTINS
+from benchmarks.suites.gsm8k import _BUILTIN_PROBLEMS as _GSM8K_BUILTINS
+from benchmarks.suites.humaneval import _BUILTIN_PROBLEMS as _HUMANEVAL_BUILTINS
+from benchmarks.suites.ifeval import _BUILTIN_PROBLEMS as _IFEVAL_BUILTINS
+from benchmarks.suites.livecodebench import _BUILTIN_PROBLEMS as _LIVECODEBENCH_BUILTINS
+from benchmarks.suites.math_bench import _BUILTIN_PROBLEMS as _MATH_BUILTINS
+from benchmarks.suites.mbpp import _BUILTIN_PROBLEMS as _MBPP_BUILTINS
+from benchmarks.suites.swe_bench import _BUILTIN_PROBLEMS as _SWE_BUILTINS
+
+
+HF_DATASET_VIEWER_BASE = "https://datasets-server.huggingface.co"
+HUMANEVAL_GZ_URL = "https://raw.githubusercontent.com/openai/human-eval/master/data/HumanEval.jsonl.gz"
+DEFAULT_DATA_DIR = Path(__file__).resolve().parent / "data"
+
+JsonFetcher = Callable[[str, dict[str, object], dict[str, str] | None, float], object]
+
+
+@dataclass
+class DownloadResult:
+    suite: str
+    rows: int
+    path: str
+    source: str
+    note: str = ""
+
+
+def fetch_bytes(url: str, timeout: float, headers: dict[str, str] | None = None) -> bytes:
+    request = urllib.request.Request(url, headers=headers or {})
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return response.read()
+
+
+def fetch_json(
+    endpoint: str,
+    params: dict[str, object],
+    headers: dict[str, str] | None,
+    timeout: float,
+) -> object:
+    query = urllib.parse.urlencode(params, doseq=True)
+    url = f"{HF_DATASET_VIEWER_BASE}/{endpoint}"
+    if query:
+        url = f"{url}?{query}"
+    raw = fetch_bytes(url, timeout=timeout, headers=headers)
+    return json.loads(raw.decode("utf-8"))
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> int:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=True) + "\n")
+    return len(rows)
+
+
+def _extract_gsm8k_answer(text: str) -> str:
+    if "####" in text:
+        text = text.split("####", 1)[1]
+    numbers = re.findall(r"-?\d[\d,]*\.?\d*", text.replace("$", ""))
+    if numbers:
+        return numbers[-1].replace(",", "")
+    return text.strip().replace(",", "")
+
+
+def _extract_math_answer(solution: str) -> str:
+    boxed_fraction = re.search(r"\\boxed\{\\frac\{([^}]+)\}\{([^}]+)\}\}", solution, flags=re.DOTALL)
+    if boxed_fraction:
+        return f"{boxed_fraction.group(1).strip()}/{boxed_fraction.group(2).strip()}"
+    boxed = re.search(r"\\boxed\{([^{}]+)\}", solution, flags=re.DOTALL)
+    value = boxed.group(1) if boxed else solution
+    value = value.strip()
+    value = value.replace("\\frac{", "").replace("}{", "/").replace("}", "")
+    value = value.replace("$", "").replace(",", "").strip()
+    fraction = re.search(r"-?\d+\s*/\s*-?\d+", value)
+    if fraction:
+        return fraction.group(0).replace(" ", "")
+    numbers = re.findall(r"-?\d+(?:/\d+)?(?:\.\d+)?", value)
+    return numbers[-1] if numbers else value
+
+
+def _fetch_hf_rows(
+    dataset: str,
+    *,
+    config_preference: tuple[str, ...] = (),
+    split_preference: tuple[str, ...] = ("test", "validation", "train"),
+    json_fetcher: JsonFetcher = fetch_json,
+    timeout: float = 60.0,
+    headers: dict[str, str] | None = None,
+) -> list[dict[str, Any]]:
+    splits_payload = json_fetcher("splits", {"dataset": dataset}, headers, timeout)
+    splits = list((splits_payload or {}).get("splits", []))  # type: ignore[assignment]
+    if not splits:
+        return []
+
+    chosen: dict[str, Any] | None = None
+    for config_name in config_preference:
+        for split_name in split_preference:
+            chosen = next(
+                (
+                    item for item in splits
+                    if item.get("config") == config_name and item.get("split") == split_name
+                ),
+                None,
+            )
+            if chosen is not None:
+                break
+        if chosen is not None:
+            break
+    if chosen is None:
+        for split_name in split_preference:
+            chosen = next((item for item in splits if item.get("split") == split_name), None)
+            if chosen is not None:
+                break
+    if chosen is None:
+        chosen = splits[0]
+
+    rows: list[dict[str, Any]] = []
+    offset = 0
+    length = 100
+    while True:
+        payload = json_fetcher(
+            "rows",
+            {
+                "dataset": chosen["dataset"],
+                "config": chosen["config"],
+                "split": chosen["split"],
+                "offset": offset,
+                "length": length,
+            },
+            headers,
+            timeout,
+        )
+        batch = [item.get("row", {}) for item in (payload or {}).get("rows", [])]  # type: ignore[union-attr]
+        rows.extend(batch)
+        total = int((payload or {}).get("num_rows_total", len(rows)))  # type: ignore[union-attr]
+        offset += len(batch)
+        if not batch or offset >= total:
+            break
+    return rows
+
+
+def _download_humaneval(output_path: Path, *, timeout: float) -> DownloadResult:
+    raw = fetch_bytes(HUMANEVAL_GZ_URL, timeout=timeout)
+    if raw[:2] == b"\x1f\x8b":
+        raw = gzip.decompress(raw)
+    lines = [json.loads(line) for line in raw.decode("utf-8").splitlines() if line.strip()]
+    rows = [
+        {
+            "task_id": item["task_id"],
+            "prompt": item["prompt"],
+            "canonical_solution": item.get("canonical_solution", ""),
+            "test": item["test"],
+            "entry_point": item["entry_point"],
+        }
+        for item in lines
+    ]
+    count = _write_jsonl(output_path, rows)
+    return DownloadResult("humaneval", count, str(output_path), "official")
+
+
+def _download_gsm8k(
+    output_path: Path,
+    *,
+    timeout: float,
+    json_fetcher: JsonFetcher = fetch_json,
+) -> DownloadResult:
+    rows = _fetch_hf_rows(
+        "openai/gsm8k",
+        config_preference=("main",),
+        split_preference=("test",),
+        json_fetcher=json_fetcher,
+        timeout=timeout,
+    )
+    normalized = [
+        {
+            "id": f"gsm8k-{index + 1:04d}",
+            "question": row["question"],
+            "answer": _extract_gsm8k_answer(str(row["answer"])),
+        }
+        for index, row in enumerate(rows)
+    ]
+    count = _write_jsonl(output_path, normalized)
+    return DownloadResult("gsm8k", count, str(output_path), "official")
+
+
+def _download_mbpp(
+    output_path: Path,
+    *,
+    timeout: float,
+    json_fetcher: JsonFetcher = fetch_json,
+) -> DownloadResult:
+    rows = _fetch_hf_rows(
+        "google-research-datasets/mbpp",
+        config_preference=("sanitized", "full"),
+        split_preference=("test", "validation"),
+        json_fetcher=json_fetcher,
+        timeout=timeout,
+    )
+    normalized = [
+        {
+            "task_id": row.get("task_id", index + 1),
+            "text": row.get("text") or row.get("prompt") or "",
+            "code": row.get("code", ""),
+            "test_list": row.get("test_list") or row.get("test_setup_code", []),
+        }
+        for index, row in enumerate(rows)
+    ]
+    count = _write_jsonl(output_path, normalized)
+    return DownloadResult("mbpp", count, str(output_path), "official")
+
+
+def _download_math(
+    output_path: Path,
+    *,
+    timeout: float,
+    json_fetcher: JsonFetcher = fetch_json,
+) -> DownloadResult:
+    rows = _fetch_hf_rows(
+        "hendrycks/competition_math",
+        config_preference=("default",),
+        split_preference=("test", "train"),
+        json_fetcher=json_fetcher,
+        timeout=timeout,
+    )
+    normalized = [
+        {
+            "id": row.get("problem_id", f"math-{index + 1:04d}"),
+            "problem": row.get("problem", ""),
+            "answer": _extract_math_answer(str(row.get("solution", row.get("answer", "")))),
+            "subject": row.get("type", row.get("subject", "unknown")),
+            "level": row.get("level", 0),
+        }
+        for index, row in enumerate(rows)
+    ]
+    count = _write_jsonl(output_path, normalized)
+    return DownloadResult("math", count, str(output_path), "official")
+
+
+def _export_builtin(output_path: Path, suite: str, rows: list[dict[str, Any]], *, source: str = "builtin", note: str = "") -> DownloadResult:
+    count = _write_jsonl(output_path, rows)
+    return DownloadResult(suite, count, str(output_path), source, note)
+
+
+def _builtin_rows(suite: str) -> list[dict[str, Any]]:
+    mapping: dict[str, list[dict[str, Any]]] = {
+        "humaneval": list(_HUMANEVAL_BUILTINS),
+        "mbpp": list(_MBPP_BUILTINS),
+        "gsm8k": list(_GSM8K_BUILTINS),
+        "math": list(_MATH_BUILTINS),
+        "swe-bench": list(_SWE_BUILTINS),
+        "aider": list(_AIDER_BUILTINS),
+        "livecodebench": list(_LIVECODEBENCH_BUILTINS),
+        "aime": list(_AIME_BUILTINS),
+        "ifeval": list(_IFEVAL_BUILTINS),
+        "bfcl": list(_BFCL_BUILTINS),
+    }
+    return list(mapping[suite])
+
+
+def prepare_suite(
+    suite: str,
+    *,
+    data_dir: Path,
+    force: bool,
+    builtin_only: bool,
+    official_only: bool,
+    timeout: float,
+) -> DownloadResult:
+    output_path = data_dir / f"{suite}.jsonl"
+    if output_path.exists() and not force:
+        lines = [line for line in output_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+        return DownloadResult(suite, len(lines), str(output_path), "existing")
+
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    builtin_only_suites = {"swe-bench", "aider", "livecodebench", "aime", "ifeval", "bfcl"}
+    official_downloaders = {
+        "humaneval": _download_humaneval,
+        "gsm8k": _download_gsm8k,
+        "mbpp": _download_mbpp,
+        "math": _download_math,
+    }
+
+    if suite in builtin_only_suites or builtin_only:
+        return _export_builtin(output_path, suite, _builtin_rows(suite))
+
+    downloader = official_downloaders.get(suite)
+    if downloader is None:
+        return _export_builtin(output_path, suite, _builtin_rows(suite))
+
+    try:
+        return downloader(output_path, timeout=timeout)
+    except Exception as exc:
+        if official_only:
+            raise
+        note = f"official download failed: {exc}"
+        return _export_builtin(
+            output_path,
+            suite,
+            _builtin_rows(suite),
+            source="builtin-fallback",
+            note=note,
+        )
+
+
+def _write_manifest(data_dir: Path, results: list[DownloadResult]) -> Path:
+    manifest_path = data_dir / "manifest.json"
+    payload = {
+        "generated_at": __import__("time").strftime("%Y-%m-%dT%H:%M:%S"),
+        "results": [asdict(item) for item in results],
+    }
+    manifest_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return manifest_path
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Download benchmark datasets for claw-code-agent.")
+    parser.add_argument("--suite", action="append", default=[], help="Suite to prepare. Can be repeated.")
+    parser.add_argument("--all", action="store_true", help="Prepare all known suites.")
+    parser.add_argument("--list", action="store_true", help="List known suites.")
+    parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR, help="Output data directory.")
+    parser.add_argument("--force", action="store_true", help="Overwrite existing files.")
+    parser.add_argument("--builtin-only", action="store_true", help="Skip official downloads and export builtins only.")
+    parser.add_argument("--official-only", action="store_true", help="Do not fall back to builtins.")
+    parser.add_argument("--timeout", type=float, default=60.0, help="Network timeout in seconds.")
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+    known = [
+        "humaneval",
+        "mbpp",
+        "gsm8k",
+        "math",
+        "swe-bench",
+        "aider",
+        "livecodebench",
+        "aime",
+        "ifeval",
+        "bfcl",
+    ]
+
+    if args.list:
+        for name in known:
+            print(name)
+        return
+
+    suites = list(args.suite)
+    if args.all:
+        suites = known
+    if not suites:
+        parser.error("specify --suite or --all")
+
+    results = [
+        prepare_suite(
+            suite,
+            data_dir=args.data_dir,
+            force=args.force,
+            builtin_only=args.builtin_only,
+            official_only=args.official_only,
+            timeout=args.timeout,
+        )
+        for suite in suites
+    ]
+    manifest = _write_manifest(args.data_dir, results)
+    print(f"Wrote {len(results)} suite files to {args.data_dir}")
+    print(f"Manifest: {manifest}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/run.py

@@ -39,6 +39,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from benchmarks.tasks.definitions import TASKS, BenchmarkTask, get_task
+from benchmarks.suites.base import make_temp_workspace
 
 
 # ---------------------------------------------------------------------------
@@ -90,7 +91,7 @@ def run_task(
     """Run a single benchmark task end-to-end."""
 
     # Create isolated temp workspace
-    workspace = tempfile.mkdtemp(prefix=f"claw_bench_{task.id}_")
+    workspace = make_temp_workspace("claw_bench", task.category, task.id)
 
     if verbose:
         print(f"  workspace: {workspace}")

benchmarks/run_suite.py

@@ -193,6 +193,10 @@ def main() -> None:
                         help="Save results to JSON file")
     parser.add_argument("--data-dir",
                         help="Directory containing dataset files (JSONL)")
+    parser.add_argument("--artifacts-dir",
+                        help="Directory where per-problem artifacts will be saved")
+    parser.add_argument("--save-passing-artifacts", action="store_true",
+                        help="Also save artifacts for passing problems")
     args = parser.parse_args()
 
     if args.list:
@@ -236,6 +240,8 @@ def main() -> None:
             limit=args.limit,
             agent_timeout=args.timeout,
             verbose=args.verbose,
+            artifacts_dir=args.artifacts_dir,
+            save_passing_artifacts=args.save_passing_artifacts,
         )
         report = suite.run_all()
         reports.append(report)

benchmarks/run_terminal_bench_local.py

@@ -0,0 +1,512 @@
+#!/usr/bin/env python3
+"""
+Run Terminal-Bench tasks locally with Apptainer and claw-code-agent.
+
+This runner is designed for cluster setups where Docker is unavailable but
+Apptainer is available and the model server runs on the same node.
+"""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import json
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import time
+import tomllib
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_TASKS_DIR = Path.home() / ".cache/harbor/tasks/packages/terminal-bench"
+DEFAULT_JOBS_DIR = Path("jobs/terminal_bench_local")
+
+
+@dataclass
+class TerminalBenchTask:
+    task_dir: Path
+    name: str
+    short_name: str
+    instruction: str
+    docker_image: str | None
+    agent_timeout_sec: float | None
+    verifier_timeout_sec: float
+    workdir: str
+    has_docker_compose: bool
+    raw_config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class LocalTrialResult:
+    task_name: str
+    short_name: str
+    passed: bool
+    reward: float | None
+    duration_sec: float
+    status: str
+    image: str | None
+    workdir: str
+    trial_dir: str
+    error: str = ""
+    agent_return_code: int | None = None
+    verifier_return_code: int | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def _load_toml(path: Path) -> dict[str, Any]:
+    return tomllib.loads(path.read_text(encoding="utf-8"))
+
+
+def strip_canary(text: str) -> str:
+    lines = text.splitlines()
+    index = 0
+    while index < len(lines):
+        stripped = lines[index].strip()
+        if stripped.startswith("<!--") and "canary" in stripped.lower():
+            index += 1
+            continue
+        if stripped.startswith("#") and "canary" in stripped.lower():
+            index += 1
+            continue
+        break
+    while index < len(lines) and not lines[index].strip():
+        index += 1
+    return "\n".join(lines[index:])
+
+
+def parse_dockerfile_workdir(dockerfile_path: Path) -> str:
+    if not dockerfile_path.exists():
+        return "/workspace"
+    workdir = "/workspace"
+    pattern = re.compile(r"^\s*WORKDIR\s+(.+?)\s*$", re.IGNORECASE)
+    for raw_line in dockerfile_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        match = pattern.match(line)
+        if not match:
+            continue
+        value = match.group(1).strip()
+        if value.startswith("["):
+            continue
+        if value.startswith('"') and value.endswith('"'):
+            value = value[1:-1]
+        workdir = value
+    return workdir
+
+
+def load_task(task_dir: Path) -> TerminalBenchTask:
+    config = _load_toml(task_dir / "task.toml")
+    task_config = config.get("task") or {}
+    environment = config.get("environment") or {}
+    agent = config.get("agent") or {}
+    verifier = config.get("verifier") or {}
+    name = task_config.get("name", task_dir.name)
+    short_name = name.split("/", 1)[-1]
+    instruction = strip_canary((task_dir / "instruction.md").read_text(encoding="utf-8"))
+    environment_dir = task_dir / "environment"
+    return TerminalBenchTask(
+        task_dir=task_dir,
+        name=name,
+        short_name=short_name,
+        instruction=instruction,
+        docker_image=environment.get("docker_image"),
+        agent_timeout_sec=agent.get("timeout_sec"),
+        verifier_timeout_sec=float(verifier.get("timeout_sec", 600.0)),
+        workdir=parse_dockerfile_workdir(environment_dir / "Dockerfile"),
+        has_docker_compose=(environment_dir / "docker-compose.yaml").exists(),
+        raw_config=config,
+    )
+
+
+def discover_tasks(root: Path) -> list[TerminalBenchTask]:
+    if (root / "task.toml").exists():
+        return [load_task(root)]
+    tasks: list[TerminalBenchTask] = []
+    for config_path in sorted(root.rglob("task.toml")):
+        try:
+            tasks.append(load_task(config_path.parent))
+        except Exception:
+            continue
+    return tasks
+
+
+def filter_tasks(
+    tasks: list[TerminalBenchTask],
+    *,
+    include_patterns: list[str],
+    exclude_patterns: list[str],
+    limit: int | None,
+) -> list[TerminalBenchTask]:
+    selected: list[TerminalBenchTask] = []
+    for task in tasks:
+        candidates = {task.name, task.short_name, task.task_dir.name}
+        if include_patterns and not any(
+            fnmatch.fnmatchcase(candidate, pattern)
+            for pattern in include_patterns
+            for candidate in candidates
+        ):
+            continue
+        if any(
+            fnmatch.fnmatchcase(candidate, pattern)
+            for pattern in exclude_patterns
+            for candidate in candidates
+        ):
+            continue
+        selected.append(task)
+    if limit is not None:
+        selected = selected[:limit]
+    return selected
+
+
+def run_shell(
+    cmd: str,
+    *,
+    timeout: float | None = None,
+    env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        cmd,
+        shell=True,
+        text=True,
+        capture_output=True,
+        timeout=timeout,
+        env=env,
+    )
+
+
+def safe_name(value: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._") or "item"
+
+
+def ensure_apptainer_image(task: TerminalBenchTask, images_dir: Path, force_pull: bool) -> Path:
+    if not task.docker_image:
+        raise ValueError("task has no docker_image")
+    images_dir.mkdir(parents=True, exist_ok=True)
+    image_path = images_dir / f"{safe_name(task.name)}.sif"
+    if image_path.exists() and not force_pull:
+        return image_path
+    cmd = f"apptainer pull --force {shlex.quote(str(image_path))} docker://{shlex.quote(task.docker_image)}"
+    result = run_shell(cmd, timeout=3600.0)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"apptainer pull failed for {task.docker_image}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    return image_path
+
+
+def seed_workspace(task: TerminalBenchTask, image_path: Path, workspace_dir: Path) -> None:
+    workspace_dir.mkdir(parents=True, exist_ok=True)
+    marker = workspace_dir / ".seed_complete"
+    if marker.exists():
+        return
+
+    copy_cmd = (
+        "set -euo pipefail; "
+        f"if [ -d {shlex.quote(task.workdir)} ]; then "
+        f"mkdir -p /mnt/workspace && cp -a {shlex.quote(task.workdir)}/. /mnt/workspace/; "
+        "fi"
+    )
+    cmd = (
+        f"apptainer exec --bind {shlex.quote(str(workspace_dir))}:/mnt/workspace:rw "
+        f"{shlex.quote(str(image_path))} "
+        f"bash -lc {shlex.quote(copy_cmd)}"
+    )
+    result = run_shell(cmd, timeout=1800.0)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"failed to seed workspace from {task.workdir}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    marker.write_text("ok\n", encoding="utf-8")
+
+
+def read_reward(verifier_dir: Path) -> tuple[float | None, dict[str, Any]]:
+    reward_txt = verifier_dir / "reward.txt"
+    reward_json = verifier_dir / "reward.json"
+    if reward_txt.exists():
+        raw = reward_txt.read_text(encoding="utf-8").strip()
+        return float(raw), {"reward_source": "reward.txt"}
+    if reward_json.exists():
+        payload = json.loads(reward_json.read_text(encoding="utf-8"))
+        reward = payload.get("reward")
+        if reward is None:
+            numeric = [value for value in payload.values() if isinstance(value, (int, float))]
+            reward = float(numeric[0]) if numeric else None
+        return (float(reward) if reward is not None else None), {"reward_source": "reward.json", "reward_payload": payload}
+    return None, {}
+
+
+def build_host_agent_command(
+    *,
+    task: TerminalBenchTask,
+    workspace_dir: Path,
+    repo_dir: Path,
+    agent_logs_dir: Path,
+) -> str:
+    instruction_file = agent_logs_dir / "instruction.txt"
+    instruction_file.write_text(task.instruction, encoding="utf-8")
+    stdout_path = agent_logs_dir / "stdout.txt"
+    stderr_path = agent_logs_dir / "stderr.txt"
+    return (
+        "set -euo pipefail; "
+        f"cd {shlex.quote(str(repo_dir))}; "
+        f"instruction=$(cat {shlex.quote(str(instruction_file))}); "
+        f"{shlex.quote(sys.executable)} -m src.main agent \"$instruction\" "
+        f"--cwd {shlex.quote(str(workspace_dir))} --allow-write --allow-shell "
+        f"> {shlex.quote(str(stdout_path))} 2> {shlex.quote(str(stderr_path))}"
+    )
+
+
+def build_verifier_exec_command(
+    *,
+    task: TerminalBenchTask,
+    image_path: Path,
+    workspace_dir: Path,
+    task_dir: Path,
+    verifier_logs_dir: Path,
+    env: dict[str, str],
+) -> str:
+    binds = [
+        f"{workspace_dir}:{task.workdir}:rw",
+        f"{task_dir / 'tests'}:/tests:ro",
+        f"{verifier_logs_dir}:/logs/verifier:rw",
+    ]
+    bind_flags = " ".join(f"--bind {shlex.quote(spec)}" for spec in binds)
+    env_flags = " ".join(
+        f"--env {shlex.quote(key)}={shlex.quote(value)}"
+        for key, value in sorted(env.items())
+    )
+    inner = (
+        "set -euo pipefail; "
+        "chmod +x /tests/test.sh; "
+        f"cd {shlex.quote(task.workdir)}; "
+        "/tests/test.sh > /logs/verifier/test-stdout.txt 2> /logs/verifier/test-stderr.txt"
+    )
+    return (
+        f"apptainer exec --cleanenv {bind_flags} {env_flags} "
+        f"--cwd {shlex.quote(task.workdir)} {shlex.quote(str(image_path))} "
+        f"bash -lc {shlex.quote(inner)}"
+    )
+
+
+def run_trial(
+    task: TerminalBenchTask,
+    *,
+    repo_dir: Path,
+    jobs_dir: Path,
+    images_dir: Path,
+    force_pull: bool,
+    keep_images: bool,
+    timeout_multiplier: float,
+) -> LocalTrialResult:
+    timestamp = time.strftime("%Y%m%d-%H%M%S")
+    trial_dir = jobs_dir / f"{timestamp}_{safe_name(task.short_name)}"
+    workspace_dir = trial_dir / "workspace"
+    agent_logs_dir = trial_dir / "agent"
+    verifier_logs_dir = trial_dir / "verifier"
+    for path in (trial_dir, workspace_dir, agent_logs_dir, verifier_logs_dir):
+        path.mkdir(parents=True, exist_ok=True)
+
+    start = time.time()
+    result = LocalTrialResult(
+        task_name=task.name,
+        short_name=task.short_name,
+        passed=False,
+        reward=None,
+        duration_sec=0.0,
+        status="pending",
+        image=task.docker_image,
+        workdir=task.workdir,
+        trial_dir=str(trial_dir),
+    )
+
+    if task.has_docker_compose:
+        result.status = "skipped"
+        result.error = "docker-compose task is not supported by the local Apptainer runner"
+        result.duration_sec = time.time() - start
+        return result
+    if not task.docker_image:
+        result.status = "skipped"
+        result.error = "task has no prebuilt docker_image in task.toml"
+        result.duration_sec = time.time() - start
+        return result
+
+    image_path = ensure_apptainer_image(task, images_dir, force_pull)
+    if not keep_images:
+        result.metadata["image_path"] = str(image_path)
+    seed_workspace(task, image_path, workspace_dir)
+
+    env = {
+        key: os.environ[key]
+        for key in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_MODEL")
+        if os.environ.get(key)
+    }
+    env.update({str(k): str(v) for k, v in (task.raw_config.get("environment") or {}).get("env", {}).items()})
+    verifier_env = {str(k): str(v) for k, v in (task.raw_config.get("verifier") or {}).get("env", {}).items()}
+
+    agent_cmd = build_host_agent_command(
+        task=task,
+        workspace_dir=workspace_dir,
+        repo_dir=repo_dir,
+        agent_logs_dir=agent_logs_dir,
+    )
+    verifier_cmd = build_verifier_exec_command(
+        task=task,
+        image_path=image_path,
+        workspace_dir=workspace_dir,
+        task_dir=task.task_dir,
+        verifier_logs_dir=verifier_logs_dir,
+        env=verifier_env,
+    )
+
+    agent_timeout = (task.agent_timeout_sec or 1800.0) * timeout_multiplier
+    verifier_timeout = task.verifier_timeout_sec * timeout_multiplier
+
+    try:
+        agent_proc = run_shell(agent_cmd, timeout=agent_timeout, env={**os.environ, **env})
+        result.agent_return_code = agent_proc.returncode
+        if agent_proc.returncode != 0:
+            result.status = "agent_failed"
+            stdout_path = agent_logs_dir / "stdout.txt"
+            stderr_path = agent_logs_dir / "stderr.txt"
+            stdout_text = stdout_path.read_text(encoding="utf-8", errors="ignore") if stdout_path.exists() else agent_proc.stdout
+            stderr_text = stderr_path.read_text(encoding="utf-8", errors="ignore") if stderr_path.exists() else agent_proc.stderr
+            result.error = (stdout_text + "\n" + stderr_text).strip()[:4000]
+            result.duration_sec = time.time() - start
+            return result
+
+        verifier_proc = run_shell(verifier_cmd, timeout=verifier_timeout)
+        result.verifier_return_code = verifier_proc.returncode
+        reward, reward_metadata = read_reward(verifier_logs_dir)
+        result.reward = reward
+        result.metadata.update(reward_metadata)
+        result.passed = verifier_proc.returncode == 0 and reward is not None and reward > 0.0
+        result.status = "passed" if result.passed else "failed"
+        if not result.passed:
+            stdout_text = (verifier_logs_dir / "test-stdout.txt").read_text(encoding="utf-8", errors="ignore") if (verifier_logs_dir / "test-stdout.txt").exists() else ""
+            stderr_text = (verifier_logs_dir / "test-stderr.txt").read_text(encoding="utf-8", errors="ignore") if (verifier_logs_dir / "test-stderr.txt").exists() else ""
+            result.error = (stdout_text + "\n" + stderr_text).strip()[:4000]
+        return result
+    except subprocess.TimeoutExpired as exc:
+        result.status = "timeout"
+        result.error = f"timeout after {exc.timeout}s"
+        return result
+    except Exception as exc:
+        result.status = "error"
+        result.error = str(exc)
+        return result
+    finally:
+        result.duration_sec = time.time() - start
+
+
+def print_report(results: list[LocalTrialResult]) -> None:
+    total = len(results)
+    passed = sum(1 for item in results if item.passed)
+    failed = sum(1 for item in results if item.status == "failed")
+    skipped = sum(1 for item in results if item.status == "skipped")
+    print()
+    print("=" * 80)
+    print("  TERMINAL-BENCH LOCAL REPORT")
+    print("=" * 80)
+    for item in results:
+        icon = "PASS ✅" if item.passed else ("SKIP ⚪" if item.status == "skipped" else "FAIL ❌")
+        print(f"  {icon:<8} {item.short_name:<32} {item.duration_sec:>7.1f}s  {item.status}")
+    print("─" * 80)
+    print(f"  Total: {total}  Passed: {passed}  Failed: {failed}  Skipped: {skipped}")
+    score = (100.0 * passed / total) if total else 0.0
+    print(f"  Score: {score:.1f}%")
+    print("─" * 80)
+
+
+def save_results(path: Path, results: list[LocalTrialResult]) -> None:
+    payload = {
+        "benchmark": "terminal-bench-local-apptainer",
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
+        "model": os.environ.get("OPENAI_MODEL", "unknown"),
+        "results": [asdict(item) for item in results],
+    }
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Run Terminal-Bench tasks locally with Apptainer and claw-code-agent.")
+    parser.add_argument("--tasks-dir", type=Path, default=DEFAULT_TASKS_DIR, help="Directory containing downloaded Harbor task packages.")
+    parser.add_argument("--include-task-name", "-i", action="append", default=[], help="Include only task names matching this glob pattern.")
+    parser.add_argument("--exclude-task-name", "-x", action="append", default=[], help="Exclude task names matching this glob pattern.")
+    parser.add_argument("--n-tasks", "-l", type=int, default=None, help="Maximum number of tasks to run after filtering.")
+    parser.add_argument("--jobs-dir", "-o", type=Path, default=DEFAULT_JOBS_DIR, help="Directory where trial outputs will be written.")
+    parser.add_argument("--images-dir", type=Path, default=DEFAULT_JOBS_DIR / "_images", help="Directory where pulled Apptainer images will be cached.")
+    parser.add_argument("--force-pull", action="store_true", help="Re-pull Apptainer images even if cached locally.")
+    parser.add_argument("--keep-images", action="store_true", help="Keep pulled SIF images in the image cache directory.")
+    parser.add_argument("--timeout-multiplier", type=float, default=1.0, help="Multiplier applied to agent and verifier timeouts.")
+    parser.add_argument("--output", type=Path, help="Optional JSON file for the run summary.")
+    parser.add_argument("--list", action="store_true", help="List discovered tasks and exit.")
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if not args.tasks_dir.exists():
+        parser.error(f"tasks directory not found: {args.tasks_dir}")
+
+    tasks = discover_tasks(args.tasks_dir)
+    tasks = filter_tasks(
+        tasks,
+        include_patterns=args.include_task_name,
+        exclude_patterns=args.exclude_task_name,
+        limit=args.n_tasks,
+    )
+
+    if args.list:
+        for task in tasks:
+            print(task.short_name)
+        return
+
+    if not tasks:
+        parser.error("no tasks matched")
+
+    repo_dir = Path(__file__).resolve().parent.parent
+    results: list[LocalTrialResult] = []
+    print()
+    print("=" * 80)
+    print("  TERMINAL-BENCH LOCAL")
+    print("=" * 80)
+    print(f"  Tasks:    {len(tasks)}")
+    print(f"  Jobs dir: {args.jobs_dir}")
+    print("=" * 80)
+    print()
+
+    for index, task in enumerate(tasks, 1):
+        print(f"[{index}/{len(tasks)}] {task.short_name}")
+        result = run_trial(
+            task,
+            repo_dir=repo_dir,
+            jobs_dir=args.jobs_dir,
+            images_dir=args.images_dir,
+            force_pull=args.force_pull,
+            keep_images=args.keep_images,
+            timeout_multiplier=args.timeout_multiplier,
+        )
+        results.append(result)
+        icon = "PASS ✅" if result.passed else ("SKIP ⚪" if result.status == "skipped" else "FAIL ❌")
+        print(f"  -> {icon}  ({result.duration_sec:.1f}s)")
+        print(f"     trial_dir: {result.trial_dir}")
+        if result.error:
+            print(f"     error: {result.error[:200]}")
+        print()
+
+    print_report(results)
+    if args.output:
+        save_results(args.output, results)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/suites/aime.py

@@ -110,6 +110,22 @@ class AIMEBenchmark(BenchmarkSuite):
             f"no explanation, no work, just the number."
         )
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        answer_path = Path(workspace) / "answer.txt"
+        if answer_path.exists():
+            return
+        numbers = re.findall(r"-?\d+", agent_output.replace(",", ""))
+        if numbers:
+            answer_path.write_text(numbers[-1] + "\n", encoding="utf-8")
+            metadata["recovered_answer_from_output"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", "unknown")
         expected = str(problem["answer"]).strip()

benchmarks/suites/base.py

@@ -1,17 +1,12 @@
 """
-Base class for all benchmark suites.
-
-Every suite implements:
-  - load_dataset()   -> download/load the evaluation dataset
-  - run_single()     -> run the agent on one problem
-  - evaluate()       -> score agent output for one problem
-  - run_all()        -> orchestrate the full benchmark
+Base class for benchmark suites and shared benchmark helpers.
 """
 
 from __future__ import annotations
 
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -23,10 +18,33 @@ from pathlib import Path
 from typing import Any
 
 
+_SAFE_COMPONENT_RE = re.compile(r"[^A-Za-z0-9_.-]+")
+
+
+def _safe_component(value: str) -> str:
+    cleaned = _SAFE_COMPONENT_RE.sub("_", value).strip("._")
+    return cleaned or "item"
+
+
+def resolve_temp_root() -> Path:
+    root = Path(tempfile.gettempdir()).expanduser().resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def make_temp_workspace(prefix: str, suite_name: str, problem_id: str) -> str:
+    temp_root = resolve_temp_root()
+    safe_prefix = _safe_component(prefix)
+    safe_suite = _safe_component(suite_name)
+    safe_problem = _safe_component(problem_id)
+    return tempfile.mkdtemp(
+        prefix=f"{safe_prefix}_{safe_suite}_{safe_problem}_",
+        dir=str(temp_root),
+    )
+
+
 @dataclass
 class BenchmarkResult:
-    """Result for a single problem in a benchmark suite."""
-
     problem_id: str
     passed: bool
     expected: str = ""
@@ -38,8 +56,6 @@ class BenchmarkResult:
 
 @dataclass
 class SuiteReport:
-    """Aggregated report for a full benchmark suite run."""
-
     suite_name: str
     total: int
     passed: int
@@ -65,12 +81,9 @@ class SuiteReport:
 
 
 class BenchmarkSuite(ABC):
-    """Abstract base for a benchmark suite."""
-
-    # Subclasses set these
     name: str = "base"
     description: str = ""
-    category: str = "general"  # coding | math | instruction-following
+    category: str = "general"
 
     def __init__(
         self,
@@ -79,6 +92,8 @@ class BenchmarkSuite(ABC):
         limit: int | None = None,
         agent_timeout: float = 300.0,
         verbose: bool = False,
+        artifacts_dir: str | None = None,
+        save_passing_artifacts: bool = False,
     ) -> None:
         self.data_dir = data_dir or str(
             Path(__file__).resolve().parent.parent / "data"
@@ -86,37 +101,28 @@ class BenchmarkSuite(ABC):
         self.limit = limit
         self.agent_timeout = agent_timeout
         self.verbose = verbose
+        self.artifacts_dir = artifacts_dir
+        self.save_passing_artifacts = save_passing_artifacts
         self.project_root = str(Path(__file__).resolve().parent.parent.parent)
 
-    # ------------------------------------------------------------------
-    # Abstract interface
-    # ------------------------------------------------------------------
-
     @abstractmethod
     def load_dataset(self) -> list[dict[str, Any]]:
-        """Return a list of problem dicts. Each must have at least an 'id' key."""
         ...
 
     @abstractmethod
     def build_prompt(self, problem: dict[str, Any]) -> str:
-        """Convert a problem dict into the instruction string sent to the agent."""
         ...
 
     @abstractmethod
-    def evaluate(
-        self, problem: dict[str, Any], workspace: str
-    ) -> BenchmarkResult:
-        """Score the agent's output for one problem.  Return a BenchmarkResult."""
+    def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         ...
 
-    # ------------------------------------------------------------------
-    # Agent execution helpers
-    # ------------------------------------------------------------------
-
     def _run_shell(
-        self, cmd: str, cwd: str, timeout: float = 30.0
+        self,
+        cmd: str,
+        cwd: str,
+        timeout: float = 30.0,
     ) -> tuple[int, str]:
-        """Run a shell command, return (exit_code, combined_output)."""
         try:
             proc = subprocess.run(
                 cmd,
@@ -133,10 +139,6 @@ class BenchmarkSuite(ABC):
             return 1, str(exc)
 
     def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
-        """Run the claw-code-agent on *instruction* inside *workspace*.
-
-        Returns (exit_code, output, duration_sec).
-        """
         import shlex
 
         agent_cmd = (
@@ -147,11 +149,13 @@ class BenchmarkSuite(ABC):
             f"--allow-shell"
         )
         if self.verbose:
-            print(f"  agent cmd: {agent_cmd[:120]}...")
+            print(f"  agent cmd: {agent_cmd[:160]}...")
 
         start = time.time()
         code, output = self._run_shell(
-            agent_cmd, cwd=self.project_root, timeout=self.agent_timeout
+            agent_cmd,
+            cwd=self.project_root,
+            timeout=self.agent_timeout,
         )
         duration = time.time() - start
 
@@ -160,12 +164,65 @@ class BenchmarkSuite(ABC):
 
         return code, output, duration
 
-    # ------------------------------------------------------------------
-    # Orchestration
-    # ------------------------------------------------------------------
+    def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
+        del problem, workspace
+
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem, workspace, agent_output, metadata
+
+    def _artifact_root(self, index: int, problem_id: str) -> Path | None:
+        if not self.artifacts_dir:
+            return None
+        root = Path(self.artifacts_dir)
+        root.mkdir(parents=True, exist_ok=True)
+        return root / f"{index:03d}_{_safe_component(problem_id)}"
+
+    def _save_artifacts(
+        self,
+        *,
+        index: int,
+        problem: dict[str, Any],
+        prompt: str,
+        agent_output: str,
+        workspace: str,
+        result: BenchmarkResult,
+        agent_exit_code: int,
+    ) -> None:
+        artifact_root = self._artifact_root(index, result.problem_id)
+        if artifact_root is None:
+            return
+        if result.passed and not self.save_passing_artifacts:
+            return
+
+        artifact_root.mkdir(parents=True, exist_ok=True)
+        (artifact_root / "problem.json").write_text(
+            json.dumps(problem, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        (artifact_root / "prompt.txt").write_text(prompt, encoding="utf-8")
+        (artifact_root / "agent_output.txt").write_text(agent_output, encoding="utf-8")
+
+        workspace_dst = artifact_root / "workspace"
+        if workspace_dst.exists():
+            shutil.rmtree(workspace_dst, ignore_errors=True)
+        shutil.copytree(workspace, workspace_dst)
+
+        result_payload = asdict(result)
+        result_payload["agent_exit_code"] = agent_exit_code
+        result_payload["workspace"] = workspace
+        (artifact_root / "result.json").write_text(
+            json.dumps(result_payload, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        result.metadata["artifact_path"] = str(artifact_root)
 
     def run_all(self) -> SuiteReport:
-        """Run the full benchmark suite and return a SuiteReport."""
         problems = self.load_dataset()
         if self.limit is not None:
             problems = problems[: self.limit]
@@ -182,25 +239,37 @@ class BenchmarkSuite(ABC):
         print("=" * 72)
         print()
 
-        all_results: list[BenchmarkResult] = []
         suite_start = time.time()
+        all_results: list[BenchmarkResult] = []
 
-        for i, problem in enumerate(problems, 1):
-            pid = problem.get("id", problem.get("task_id", f"problem-{i}"))
-            print(f"[{i}/{len(problems)}] {pid}")
+        for index, problem in enumerate(problems, 1):
+            pid = str(problem.get("id", problem.get("task_id", f"problem-{index}")))
+            print(f"[{index}/{len(problems)}] {pid}")
 
-            workspace = tempfile.mkdtemp(prefix=f"claw_{self.name}_{pid}_")
+            workspace = make_temp_workspace("claw", self.name, pid)
+            prompt = ""
+            agent_output = ""
+            agent_exit_code = -1
             try:
-                # Prepare workspace
                 self.setup_workspace(problem, workspace)
-
-                # Build prompt and run agent
                 prompt = self.build_prompt(problem)
-                _code, _output, duration = self.run_agent(prompt, workspace)
+                agent_exit_code, agent_output, duration = self.run_agent(prompt, workspace)
 
-                # Evaluate
+                result_metadata: dict[str, Any] = {"agent_exit_code": agent_exit_code}
+                self.recover_output_files(problem, workspace, agent_output, result_metadata)
                 result = self.evaluate(problem, workspace)
                 result.duration_sec = duration
+                result.metadata.update(result_metadata)
+
+                self._save_artifacts(
+                    index=index,
+                    problem=problem,
+                    prompt=prompt,
+                    agent_output=agent_output,
+                    workspace=workspace,
+                    result=result,
+                    agent_exit_code=agent_exit_code,
+                )
 
                 status = "PASS ✅" if result.passed else "FAIL ❌"
                 print(f"  -> {status}  ({duration:.1f}s)")
@@ -209,6 +278,16 @@ class BenchmarkSuite(ABC):
                     problem_id=pid,
                     passed=False,
                     error=str(exc),
+                    metadata={"agent_exit_code": agent_exit_code},
+                )
+                self._save_artifacts(
+                    index=index,
+                    problem=problem,
+                    prompt=prompt,
+                    agent_output=agent_output,
+                    workspace=workspace,
+                    result=result,
+                    agent_exit_code=agent_exit_code,
                 )
                 print(f"  -> ERROR ❌  {exc}")
             finally:
@@ -218,9 +297,8 @@ class BenchmarkSuite(ABC):
             print()
 
         suite_duration = time.time() - suite_start
-        passed = sum(1 for r in all_results if r.passed)
+        passed = sum(1 for item in all_results if item.passed)
         total = len(all_results)
-
         report = SuiteReport(
             suite_name=self.name,
             total=total,
@@ -232,19 +310,9 @@ class BenchmarkSuite(ABC):
             results=all_results,
             timestamp=time.strftime("%Y-%m-%dT%H:%M:%S"),
         )
-
         self._print_report(report)
         return report
 
-    def setup_workspace(
-        self, problem: dict[str, Any], workspace: str
-    ) -> None:
-        """Optional: prepare files in workspace before the agent runs."""
-
-    # ------------------------------------------------------------------
-    # Reporting
-    # ------------------------------------------------------------------
-
     @staticmethod
     def _print_report(report: SuiteReport) -> None:
         print()
@@ -252,9 +320,9 @@ class BenchmarkSuite(ABC):
         print(f"  {report.suite_name} — RESULTS")
         print("=" * 72)
         print()
-        for r in report.results:
-            icon = "✅" if r.passed else "❌"
-            print(f"  {icon} {r.problem_id:<40} {r.duration_sec:.1f}s")
+        for result in report.results:
+            icon = "✅" if result.passed else "❌"
+            print(f"  {icon} {result.problem_id:<40} {result.duration_sec:.1f}s")
         print()
         print("─" * 72)
         print(
@@ -267,7 +335,6 @@ class BenchmarkSuite(ABC):
 
     @staticmethod
     def save_report(report: SuiteReport, path: str) -> None:
-        """Persist a SuiteReport to JSON."""
-        with open(path, "w") as fh:
+        with open(path, "w", encoding="utf-8") as fh:
             json.dump(report.to_dict(), fh, indent=2)
         print(f"  Report saved to {path}")

benchmarks/suites/gsm8k.py

@@ -141,6 +141,22 @@ class GSM8KBenchmark(BenchmarkSuite):
             f"Save just the number to answer.txt — no units, no explanation."
         )
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        answer_path = Path(workspace) / "answer.txt"
+        if answer_path.exists():
+            return
+        number = _extract_number(agent_output)
+        if number is not None:
+            answer_path.write_text(number + "\n", encoding="utf-8")
+            metadata["recovered_answer_from_output"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", "unknown")
         expected = str(problem["answer"])

benchmarks/suites/humaneval.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import subprocess
 import sys
 import textwrap
@@ -213,19 +214,36 @@ class HumanEvalBenchmark(BenchmarkSuite):
         # Write the test harness so we can verify after the agent runs
         test_code = problem["test"]
         entry = problem["entry_point"]
-        harness = textwrap.dedent(f"""\
-            import sys
-            sys.path.insert(0, ".")
-            from solution import {entry}
-
-            {test_code}
-
-            check({entry})
-            print("ALL_TESTS_PASSED")
-        """)
+        harness = (
+            "import sys\n"
+            'sys.path.insert(0, ".")\n'
+            f"from solution import {entry}\n\n"
+            f"{test_code}\n\n"
+            f"check({entry})\n"
+            'print("ALL_TESTS_PASSED")\n'
+        )
         with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
             fh.write(harness)
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        solution_path = Path(workspace) / "solution.py"
+        if solution_path.exists():
+            return
+        blocks = re.findall(r"```(?:python)?\s*(.*?)```", agent_output, flags=re.DOTALL | re.IGNORECASE)
+        for block in blocks:
+            candidate = block.strip()
+            if "def " in candidate or "import " in candidate or "from " in candidate:
+                solution_path.write_text(candidate.rstrip() + "\n", encoding="utf-8")
+                metadata["recovered_solution_from_output"] = True
+                return
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("task_id", problem.get("id", "unknown"))
         sol = os.path.join(workspace, "solution.py")

benchmarks/suites/ifeval.py

@@ -215,6 +215,22 @@ class IFEvalBenchmark(BenchmarkSuite):
     def build_prompt(self, problem: dict[str, Any]) -> str:
         return problem["instruction"]
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        output_path = Path(workspace) / "output.txt"
+        if output_path.exists():
+            return
+        text = agent_output.strip()
+        if text:
+            output_path.write_text(text + "\n", encoding="utf-8")
+            metadata["recovered_output_from_agent_text"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", "unknown")
         checks = problem.get("checks", [])

benchmarks/suites/math_bench.py

@@ -182,6 +182,22 @@ class MATHBenchmark(BenchmarkSuite):
             f"Save just the answer to answer.txt — no explanation, no work, just the answer."
         )
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        answer_path = Path(workspace) / "answer.txt"
+        if answer_path.exists():
+            return
+        answer = _normalize_answer(agent_output)
+        if answer:
+            answer_path.write_text(answer + "\n", encoding="utf-8")
+            metadata["recovered_answer_from_output"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", str(problem.get("task_id", "unknown")))
         expected_raw = str(problem["answer"])

benchmarks/suites/mbpp.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import sys
 import textwrap
 from pathlib import Path
@@ -213,18 +214,35 @@ class MBPPBenchmark(BenchmarkSuite):
     def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
         tests = problem.get("test_list", [])
         test_code = "\n".join(tests)
-        harness = textwrap.dedent(f"""\
-            import sys
-            sys.path.insert(0, ".")
-            from solution import *
-
-            {test_code}
-
-            print("ALL_TESTS_PASSED")
-        """)
+        harness = (
+            "import sys\n"
+            'sys.path.insert(0, ".")\n'
+            "from solution import *\n\n"
+            f"{test_code}\n\n"
+            'print("ALL_TESTS_PASSED")\n'
+        )
         with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
             fh.write(harness)
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        solution_path = Path(workspace) / "solution.py"
+        if solution_path.exists():
+            return
+        blocks = re.findall(r"```(?:python)?\s*(.*?)```", agent_output, flags=re.DOTALL | re.IGNORECASE)
+        for block in blocks:
+            candidate = block.strip()
+            if "def " in candidate or "import " in candidate or "class " in candidate:
+                solution_path.write_text(candidate.rstrip() + "\n", encoding="utf-8")
+                metadata["recovered_solution_from_output"] = True
+                return
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = str(problem.get("task_id", "unknown"))
         sol = os.path.join(workspace, "solution.py")