update the codebase and clean up it

.gitignore

@@ -22,6 +22,13 @@ archive/
 .env
 .env.*
 
+# Local benchmark outputs
+benchmark_artifacts/
+jobs/
+output_terminal/
+tb2*.json
+humaneval_results.json
+
 
 test_cases
 e-commerce

benchmarks/download_datasets.py

@@ -0,0 +1,391 @@
+#!/usr/bin/env python3
+"""
+Download or export benchmark datasets into benchmarks/data.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import json
+import re
+import urllib.parse
+import urllib.request
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from benchmarks.suites.aider import _BUILTIN_PROBLEMS as _AIDER_BUILTINS
+from benchmarks.suites.aime import _BUILTIN_PROBLEMS as _AIME_BUILTINS
+from benchmarks.suites.bfcl import _BUILTIN_PROBLEMS as _BFCL_BUILTINS
+from benchmarks.suites.gsm8k import _BUILTIN_PROBLEMS as _GSM8K_BUILTINS
+from benchmarks.suites.humaneval import _BUILTIN_PROBLEMS as _HUMANEVAL_BUILTINS
+from benchmarks.suites.ifeval import _BUILTIN_PROBLEMS as _IFEVAL_BUILTINS
+from benchmarks.suites.livecodebench import _BUILTIN_PROBLEMS as _LIVECODEBENCH_BUILTINS
+from benchmarks.suites.math_bench import _BUILTIN_PROBLEMS as _MATH_BUILTINS
+from benchmarks.suites.mbpp import _BUILTIN_PROBLEMS as _MBPP_BUILTINS
+from benchmarks.suites.swe_bench import _BUILTIN_PROBLEMS as _SWE_BUILTINS
+
+
+HF_DATASET_VIEWER_BASE = "https://datasets-server.huggingface.co"
+HUMANEVAL_GZ_URL = "https://raw.githubusercontent.com/openai/human-eval/master/data/HumanEval.jsonl.gz"
+DEFAULT_DATA_DIR = Path(__file__).resolve().parent / "data"
+
+JsonFetcher = Callable[[str, dict[str, object], dict[str, str] | None, float], object]
+
+
+@dataclass
+class DownloadResult:
+    suite: str
+    rows: int
+    path: str
+    source: str
+    note: str = ""
+
+
+def fetch_bytes(url: str, timeout: float, headers: dict[str, str] | None = None) -> bytes:
+    request = urllib.request.Request(url, headers=headers or {})
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return response.read()
+
+
+def fetch_json(
+    endpoint: str,
+    params: dict[str, object],
+    headers: dict[str, str] | None,
+    timeout: float,
+) -> object:
+    query = urllib.parse.urlencode(params, doseq=True)
+    url = f"{HF_DATASET_VIEWER_BASE}/{endpoint}"
+    if query:
+        url = f"{url}?{query}"
+    raw = fetch_bytes(url, timeout=timeout, headers=headers)
+    return json.loads(raw.decode("utf-8"))
+
+
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> int:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=True) + "\n")
+    return len(rows)
+
+
+def _extract_gsm8k_answer(text: str) -> str:
+    if "####" in text:
+        text = text.split("####", 1)[1]
+    numbers = re.findall(r"-?\d[\d,]*\.?\d*", text.replace("$", ""))
+    if numbers:
+        return numbers[-1].replace(",", "")
+    return text.strip().replace(",", "")
+
+
+def _extract_math_answer(solution: str) -> str:
+    boxed_fraction = re.search(r"\\boxed\{\\frac\{([^}]+)\}\{([^}]+)\}\}", solution, flags=re.DOTALL)
+    if boxed_fraction:
+        return f"{boxed_fraction.group(1).strip()}/{boxed_fraction.group(2).strip()}"
+    boxed = re.search(r"\\boxed\{([^{}]+)\}", solution, flags=re.DOTALL)
+    value = boxed.group(1) if boxed else solution
+    value = value.strip()
+    value = value.replace("\\frac{", "").replace("}{", "/").replace("}", "")
+    value = value.replace("$", "").replace(",", "").strip()
+    fraction = re.search(r"-?\d+\s*/\s*-?\d+", value)
+    if fraction:
+        return fraction.group(0).replace(" ", "")
+    numbers = re.findall(r"-?\d+(?:/\d+)?(?:\.\d+)?", value)
+    return numbers[-1] if numbers else value
+
+
+def _fetch_hf_rows(
+    dataset: str,
+    *,
+    config_preference: tuple[str, ...] = (),
+    split_preference: tuple[str, ...] = ("test", "validation", "train"),
+    json_fetcher: JsonFetcher = fetch_json,
+    timeout: float = 60.0,
+    headers: dict[str, str] | None = None,
+) -> list[dict[str, Any]]:
+    splits_payload = json_fetcher("splits", {"dataset": dataset}, headers, timeout)
+    splits = list((splits_payload or {}).get("splits", []))  # type: ignore[assignment]
+    if not splits:
+        return []
+
+    chosen: dict[str, Any] | None = None
+    for config_name in config_preference:
+        for split_name in split_preference:
+            chosen = next(
+                (
+                    item for item in splits
+                    if item.get("config") == config_name and item.get("split") == split_name
+                ),
+                None,
+            )
+            if chosen is not None:
+                break
+        if chosen is not None:
+            break
+    if chosen is None:
+        for split_name in split_preference:
+            chosen = next((item for item in splits if item.get("split") == split_name), None)
+            if chosen is not None:
+                break
+    if chosen is None:
+        chosen = splits[0]
+
+    rows: list[dict[str, Any]] = []
+    offset = 0
+    length = 100
+    while True:
+        payload = json_fetcher(
+            "rows",
+            {
+                "dataset": chosen["dataset"],
+                "config": chosen["config"],
+                "split": chosen["split"],
+                "offset": offset,
+                "length": length,
+            },
+            headers,
+            timeout,
+        )
+        batch = [item.get("row", {}) for item in (payload or {}).get("rows", [])]  # type: ignore[union-attr]
+        rows.extend(batch)
+        total = int((payload or {}).get("num_rows_total", len(rows)))  # type: ignore[union-attr]
+        offset += len(batch)
+        if not batch or offset >= total:
+            break
+    return rows
+
+
+def _download_humaneval(output_path: Path, *, timeout: float) -> DownloadResult:
+    raw = fetch_bytes(HUMANEVAL_GZ_URL, timeout=timeout)
+    if raw[:2] == b"\x1f\x8b":
+        raw = gzip.decompress(raw)
+    lines = [json.loads(line) for line in raw.decode("utf-8").splitlines() if line.strip()]
+    rows = [
+        {
+            "task_id": item["task_id"],
+            "prompt": item["prompt"],
+            "canonical_solution": item.get("canonical_solution", ""),
+            "test": item["test"],
+            "entry_point": item["entry_point"],
+        }
+        for item in lines
+    ]
+    count = _write_jsonl(output_path, rows)
+    return DownloadResult("humaneval", count, str(output_path), "official")
+
+
+def _download_gsm8k(
+    output_path: Path,
+    *,
+    timeout: float,
+    json_fetcher: JsonFetcher = fetch_json,
+) -> DownloadResult:
+    rows = _fetch_hf_rows(
+        "openai/gsm8k",
+        config_preference=("main",),
+        split_preference=("test",),
+        json_fetcher=json_fetcher,
+        timeout=timeout,
+    )
+    normalized = [
+        {
+            "id": f"gsm8k-{index + 1:04d}",
+            "question": row["question"],
+            "answer": _extract_gsm8k_answer(str(row["answer"])),
+        }
+        for index, row in enumerate(rows)
+    ]
+    count = _write_jsonl(output_path, normalized)
+    return DownloadResult("gsm8k", count, str(output_path), "official")
+
+
+def _download_mbpp(
+    output_path: Path,
+    *,
+    timeout: float,
+    json_fetcher: JsonFetcher = fetch_json,
+) -> DownloadResult:
+    rows = _fetch_hf_rows(
+        "google-research-datasets/mbpp",
+        config_preference=("sanitized", "full"),
+        split_preference=("test", "validation"),
+        json_fetcher=json_fetcher,
+        timeout=timeout,
+    )
+    normalized = [
+        {
+            "task_id": row.get("task_id", index + 1),
+            "text": row.get("text") or row.get("prompt") or "",
+            "code": row.get("code", ""),
+            "test_list": row.get("test_list") or row.get("test_setup_code", []),
+        }
+        for index, row in enumerate(rows)
+    ]
+    count = _write_jsonl(output_path, normalized)
+    return DownloadResult("mbpp", count, str(output_path), "official")
+
+
+def _download_math(
+    output_path: Path,
+    *,
+    timeout: float,
+    json_fetcher: JsonFetcher = fetch_json,
+) -> DownloadResult:
+    rows = _fetch_hf_rows(
+        "hendrycks/competition_math",
+        config_preference=("default",),
+        split_preference=("test", "train"),
+        json_fetcher=json_fetcher,
+        timeout=timeout,
+    )
+    normalized = [
+        {
+            "id": row.get("problem_id", f"math-{index + 1:04d}"),
+            "problem": row.get("problem", ""),
+            "answer": _extract_math_answer(str(row.get("solution", row.get("answer", "")))),
+            "subject": row.get("type", row.get("subject", "unknown")),
+            "level": row.get("level", 0),
+        }
+        for index, row in enumerate(rows)
+    ]
+    count = _write_jsonl(output_path, normalized)
+    return DownloadResult("math", count, str(output_path), "official")
+
+
+def _export_builtin(output_path: Path, suite: str, rows: list[dict[str, Any]], *, source: str = "builtin", note: str = "") -> DownloadResult:
+    count = _write_jsonl(output_path, rows)
+    return DownloadResult(suite, count, str(output_path), source, note)
+
+
+def _builtin_rows(suite: str) -> list[dict[str, Any]]:
+    mapping: dict[str, list[dict[str, Any]]] = {
+        "humaneval": list(_HUMANEVAL_BUILTINS),
+        "mbpp": list(_MBPP_BUILTINS),
+        "gsm8k": list(_GSM8K_BUILTINS),
+        "math": list(_MATH_BUILTINS),
+        "swe-bench": list(_SWE_BUILTINS),
+        "aider": list(_AIDER_BUILTINS),
+        "livecodebench": list(_LIVECODEBENCH_BUILTINS),
+        "aime": list(_AIME_BUILTINS),
+        "ifeval": list(_IFEVAL_BUILTINS),
+        "bfcl": list(_BFCL_BUILTINS),
+    }
+    return list(mapping[suite])
+
+
+def prepare_suite(
+    suite: str,
+    *,
+    data_dir: Path,
+    force: bool,
+    builtin_only: bool,
+    official_only: bool,
+    timeout: float,
+) -> DownloadResult:
+    output_path = data_dir / f"{suite}.jsonl"
+    if output_path.exists() and not force:
+        lines = [line for line in output_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+        return DownloadResult(suite, len(lines), str(output_path), "existing")
+
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    builtin_only_suites = {"swe-bench", "aider", "livecodebench", "aime", "ifeval", "bfcl"}
+    official_downloaders = {
+        "humaneval": _download_humaneval,
+        "gsm8k": _download_gsm8k,
+        "mbpp": _download_mbpp,
+        "math": _download_math,
+    }
+
+    if suite in builtin_only_suites or builtin_only:
+        return _export_builtin(output_path, suite, _builtin_rows(suite))
+
+    downloader = official_downloaders.get(suite)
+    if downloader is None:
+        return _export_builtin(output_path, suite, _builtin_rows(suite))
+
+    try:
+        return downloader(output_path, timeout=timeout)
+    except Exception as exc:
+        if official_only:
+            raise
+        note = f"official download failed: {exc}"
+        return _export_builtin(
+            output_path,
+            suite,
+            _builtin_rows(suite),
+            source="builtin-fallback",
+            note=note,
+        )
+
+
+def _write_manifest(data_dir: Path, results: list[DownloadResult]) -> Path:
+    manifest_path = data_dir / "manifest.json"
+    payload = {
+        "generated_at": __import__("time").strftime("%Y-%m-%dT%H:%M:%S"),
+        "results": [asdict(item) for item in results],
+    }
+    manifest_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return manifest_path
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Download benchmark datasets for claw-code-agent.")
+    parser.add_argument("--suite", action="append", default=[], help="Suite to prepare. Can be repeated.")
+    parser.add_argument("--all", action="store_true", help="Prepare all known suites.")
+    parser.add_argument("--list", action="store_true", help="List known suites.")
+    parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR, help="Output data directory.")
+    parser.add_argument("--force", action="store_true", help="Overwrite existing files.")
+    parser.add_argument("--builtin-only", action="store_true", help="Skip official downloads and export builtins only.")
+    parser.add_argument("--official-only", action="store_true", help="Do not fall back to builtins.")
+    parser.add_argument("--timeout", type=float, default=60.0, help="Network timeout in seconds.")
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+    known = [
+        "humaneval",
+        "mbpp",
+        "gsm8k",
+        "math",
+        "swe-bench",
+        "aider",
+        "livecodebench",
+        "aime",
+        "ifeval",
+        "bfcl",
+    ]
+
+    if args.list:
+        for name in known:
+            print(name)
+        return
+
+    suites = list(args.suite)
+    if args.all:
+        suites = known
+    if not suites:
+        parser.error("specify --suite or --all")
+
+    results = [
+        prepare_suite(
+            suite,
+            data_dir=args.data_dir,
+            force=args.force,
+            builtin_only=args.builtin_only,
+            official_only=args.official_only,
+            timeout=args.timeout,
+        )
+        for suite in suites
+    ]
+    manifest = _write_manifest(args.data_dir, results)
+    print(f"Wrote {len(results)} suite files to {args.data_dir}")
+    print(f"Manifest: {manifest}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/run.py

@@ -39,6 +39,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from benchmarks.tasks.definitions import TASKS, BenchmarkTask, get_task
+from benchmarks.suites.base import make_temp_workspace
 
 
 # ---------------------------------------------------------------------------
@@ -90,7 +91,7 @@ def run_task(
     """Run a single benchmark task end-to-end."""
 
     # Create isolated temp workspace
-    workspace = tempfile.mkdtemp(prefix=f"claw_bench_{task.id}_")
+    workspace = make_temp_workspace("claw_bench", task.category, task.id)
 
     if verbose:
         print(f"  workspace: {workspace}")

benchmarks/run_suite.py

@@ -193,6 +193,10 @@ def main() -> None:
                         help="Save results to JSON file")
     parser.add_argument("--data-dir",
                         help="Directory containing dataset files (JSONL)")
+    parser.add_argument("--artifacts-dir",
+                        help="Directory where per-problem artifacts will be saved")
+    parser.add_argument("--save-passing-artifacts", action="store_true",
+                        help="Also save artifacts for passing problems")
     args = parser.parse_args()
 
     if args.list:
@@ -236,6 +240,8 @@ def main() -> None:
             limit=args.limit,
             agent_timeout=args.timeout,
             verbose=args.verbose,
+            artifacts_dir=args.artifacts_dir,
+            save_passing_artifacts=args.save_passing_artifacts,
         )
         report = suite.run_all()
         reports.append(report)

benchmarks/run_terminal_bench_local.py

@@ -0,0 +1,512 @@
+#!/usr/bin/env python3
+"""
+Run Terminal-Bench tasks locally with Apptainer and claw-code-agent.
+
+This runner is designed for cluster setups where Docker is unavailable but
+Apptainer is available and the model server runs on the same node.
+"""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import json
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import time
+import tomllib
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_TASKS_DIR = Path.home() / ".cache/harbor/tasks/packages/terminal-bench"
+DEFAULT_JOBS_DIR = Path("jobs/terminal_bench_local")
+
+
+@dataclass
+class TerminalBenchTask:
+    task_dir: Path
+    name: str
+    short_name: str
+    instruction: str
+    docker_image: str | None
+    agent_timeout_sec: float | None
+    verifier_timeout_sec: float
+    workdir: str
+    has_docker_compose: bool
+    raw_config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class LocalTrialResult:
+    task_name: str
+    short_name: str
+    passed: bool
+    reward: float | None
+    duration_sec: float
+    status: str
+    image: str | None
+    workdir: str
+    trial_dir: str
+    error: str = ""
+    agent_return_code: int | None = None
+    verifier_return_code: int | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def _load_toml(path: Path) -> dict[str, Any]:
+    return tomllib.loads(path.read_text(encoding="utf-8"))
+
+
+def strip_canary(text: str) -> str:
+    lines = text.splitlines()
+    index = 0
+    while index < len(lines):
+        stripped = lines[index].strip()
+        if stripped.startswith("<!--") and "canary" in stripped.lower():
+            index += 1
+            continue
+        if stripped.startswith("#") and "canary" in stripped.lower():
+            index += 1
+            continue
+        break
+    while index < len(lines) and not lines[index].strip():
+        index += 1
+    return "\n".join(lines[index:])
+
+
+def parse_dockerfile_workdir(dockerfile_path: Path) -> str:
+    if not dockerfile_path.exists():
+        return "/workspace"
+    workdir = "/workspace"
+    pattern = re.compile(r"^\s*WORKDIR\s+(.+?)\s*$", re.IGNORECASE)
+    for raw_line in dockerfile_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        match = pattern.match(line)
+        if not match:
+            continue
+        value = match.group(1).strip()
+        if value.startswith("["):
+            continue
+        if value.startswith('"') and value.endswith('"'):
+            value = value[1:-1]
+        workdir = value
+    return workdir
+
+
+def load_task(task_dir: Path) -> TerminalBenchTask:
+    config = _load_toml(task_dir / "task.toml")
+    task_config = config.get("task") or {}
+    environment = config.get("environment") or {}
+    agent = config.get("agent") or {}
+    verifier = config.get("verifier") or {}
+    name = task_config.get("name", task_dir.name)
+    short_name = name.split("/", 1)[-1]
+    instruction = strip_canary((task_dir / "instruction.md").read_text(encoding="utf-8"))
+    environment_dir = task_dir / "environment"
+    return TerminalBenchTask(
+        task_dir=task_dir,
+        name=name,
+        short_name=short_name,
+        instruction=instruction,
+        docker_image=environment.get("docker_image"),
+        agent_timeout_sec=agent.get("timeout_sec"),
+        verifier_timeout_sec=float(verifier.get("timeout_sec", 600.0)),
+        workdir=parse_dockerfile_workdir(environment_dir / "Dockerfile"),
+        has_docker_compose=(environment_dir / "docker-compose.yaml").exists(),
+        raw_config=config,
+    )
+
+
+def discover_tasks(root: Path) -> list[TerminalBenchTask]:
+    if (root / "task.toml").exists():
+        return [load_task(root)]
+    tasks: list[TerminalBenchTask] = []
+    for config_path in sorted(root.rglob("task.toml")):
+        try:
+            tasks.append(load_task(config_path.parent))
+        except Exception:
+            continue
+    return tasks
+
+
+def filter_tasks(
+    tasks: list[TerminalBenchTask],
+    *,
+    include_patterns: list[str],
+    exclude_patterns: list[str],
+    limit: int | None,
+) -> list[TerminalBenchTask]:
+    selected: list[TerminalBenchTask] = []
+    for task in tasks:
+        candidates = {task.name, task.short_name, task.task_dir.name}
+        if include_patterns and not any(
+            fnmatch.fnmatchcase(candidate, pattern)
+            for pattern in include_patterns
+            for candidate in candidates
+        ):
+            continue
+        if any(
+            fnmatch.fnmatchcase(candidate, pattern)
+            for pattern in exclude_patterns
+            for candidate in candidates
+        ):
+            continue
+        selected.append(task)
+    if limit is not None:
+        selected = selected[:limit]
+    return selected
+
+
+def run_shell(
+    cmd: str,
+    *,
+    timeout: float | None = None,
+    env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        cmd,
+        shell=True,
+        text=True,
+        capture_output=True,
+        timeout=timeout,
+        env=env,
+    )
+
+
+def safe_name(value: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._") or "item"
+
+
+def ensure_apptainer_image(task: TerminalBenchTask, images_dir: Path, force_pull: bool) -> Path:
+    if not task.docker_image:
+        raise ValueError("task has no docker_image")
+    images_dir.mkdir(parents=True, exist_ok=True)
+    image_path = images_dir / f"{safe_name(task.name)}.sif"
+    if image_path.exists() and not force_pull:
+        return image_path
+    cmd = f"apptainer pull --force {shlex.quote(str(image_path))} docker://{shlex.quote(task.docker_image)}"
+    result = run_shell(cmd, timeout=3600.0)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"apptainer pull failed for {task.docker_image}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    return image_path
+
+
+def seed_workspace(task: TerminalBenchTask, image_path: Path, workspace_dir: Path) -> None:
+    workspace_dir.mkdir(parents=True, exist_ok=True)
+    marker = workspace_dir / ".seed_complete"
+    if marker.exists():
+        return
+
+    copy_cmd = (
+        "set -euo pipefail; "
+        f"if [ -d {shlex.quote(task.workdir)} ]; then "
+        f"mkdir -p /mnt/workspace && cp -a {shlex.quote(task.workdir)}/. /mnt/workspace/; "
+        "fi"
+    )
+    cmd = (
+        f"apptainer exec --bind {shlex.quote(str(workspace_dir))}:/mnt/workspace:rw "
+        f"{shlex.quote(str(image_path))} "
+        f"bash -lc {shlex.quote(copy_cmd)}"
+    )
+    result = run_shell(cmd, timeout=1800.0)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"failed to seed workspace from {task.workdir}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    marker.write_text("ok\n", encoding="utf-8")
+
+
+def read_reward(verifier_dir: Path) -> tuple[float | None, dict[str, Any]]:
+    reward_txt = verifier_dir / "reward.txt"
+    reward_json = verifier_dir / "reward.json"
+    if reward_txt.exists():
+        raw = reward_txt.read_text(encoding="utf-8").strip()
+        return float(raw), {"reward_source": "reward.txt"}
+    if reward_json.exists():
+        payload = json.loads(reward_json.read_text(encoding="utf-8"))
+        reward = payload.get("reward")
+        if reward is None:
+            numeric = [value for value in payload.values() if isinstance(value, (int, float))]
+            reward = float(numeric[0]) if numeric else None
+        return (float(reward) if reward is not None else None), {"reward_source": "reward.json", "reward_payload": payload}
+    return None, {}
+
+
+def build_host_agent_command(
+    *,
+    task: TerminalBenchTask,
+    workspace_dir: Path,
+    repo_dir: Path,
+    agent_logs_dir: Path,
+) -> str:
+    instruction_file = agent_logs_dir / "instruction.txt"
+    instruction_file.write_text(task.instruction, encoding="utf-8")
+    stdout_path = agent_logs_dir / "stdout.txt"
+    stderr_path = agent_logs_dir / "stderr.txt"
+    return (
+        "set -euo pipefail; "
+        f"cd {shlex.quote(str(repo_dir))}; "
+        f"instruction=$(cat {shlex.quote(str(instruction_file))}); "
+        f"{shlex.quote(sys.executable)} -m src.main agent \"$instruction\" "
+        f"--cwd {shlex.quote(str(workspace_dir))} --allow-write --allow-shell "
+        f"> {shlex.quote(str(stdout_path))} 2> {shlex.quote(str(stderr_path))}"
+    )
+
+
+def build_verifier_exec_command(
+    *,
+    task: TerminalBenchTask,
+    image_path: Path,
+    workspace_dir: Path,
+    task_dir: Path,
+    verifier_logs_dir: Path,
+    env: dict[str, str],
+) -> str:
+    binds = [
+        f"{workspace_dir}:{task.workdir}:rw",
+        f"{task_dir / 'tests'}:/tests:ro",
+        f"{verifier_logs_dir}:/logs/verifier:rw",
+    ]
+    bind_flags = " ".join(f"--bind {shlex.quote(spec)}" for spec in binds)
+    env_flags = " ".join(
+        f"--env {shlex.quote(key)}={shlex.quote(value)}"
+        for key, value in sorted(env.items())
+    )
+    inner = (
+        "set -euo pipefail; "
+        "chmod +x /tests/test.sh; "
+        f"cd {shlex.quote(task.workdir)}; "
+        "/tests/test.sh > /logs/verifier/test-stdout.txt 2> /logs/verifier/test-stderr.txt"
+    )
+    return (
+        f"apptainer exec --cleanenv {bind_flags} {env_flags} "
+        f"--cwd {shlex.quote(task.workdir)} {shlex.quote(str(image_path))} "
+        f"bash -lc {shlex.quote(inner)}"
+    )
+
+
+def run_trial(
+    task: TerminalBenchTask,
+    *,
+    repo_dir: Path,
+    jobs_dir: Path,
+    images_dir: Path,
+    force_pull: bool,
+    keep_images: bool,
+    timeout_multiplier: float,
+) -> LocalTrialResult:
+    timestamp = time.strftime("%Y%m%d-%H%M%S")
+    trial_dir = jobs_dir / f"{timestamp}_{safe_name(task.short_name)}"
+    workspace_dir = trial_dir / "workspace"
+    agent_logs_dir = trial_dir / "agent"
+    verifier_logs_dir = trial_dir / "verifier"
+    for path in (trial_dir, workspace_dir, agent_logs_dir, verifier_logs_dir):
+        path.mkdir(parents=True, exist_ok=True)
+
+    start = time.time()
+    result = LocalTrialResult(
+        task_name=task.name,
+        short_name=task.short_name,
+        passed=False,
+        reward=None,
+        duration_sec=0.0,
+        status="pending",
+        image=task.docker_image,
+        workdir=task.workdir,
+        trial_dir=str(trial_dir),
+    )
+
+    if task.has_docker_compose:
+        result.status = "skipped"
+        result.error = "docker-compose task is not supported by the local Apptainer runner"
+        result.duration_sec = time.time() - start
+        return result
+    if not task.docker_image:
+        result.status = "skipped"
+        result.error = "task has no prebuilt docker_image in task.toml"
+        result.duration_sec = time.time() - start
+        return result
+
+    image_path = ensure_apptainer_image(task, images_dir, force_pull)
+    if not keep_images:
+        result.metadata["image_path"] = str(image_path)
+    seed_workspace(task, image_path, workspace_dir)
+
+    env = {
+        key: os.environ[key]
+        for key in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_MODEL")
+        if os.environ.get(key)
+    }
+    env.update({str(k): str(v) for k, v in (task.raw_config.get("environment") or {}).get("env", {}).items()})
+    verifier_env = {str(k): str(v) for k, v in (task.raw_config.get("verifier") or {}).get("env", {}).items()}
+
+    agent_cmd = build_host_agent_command(
+        task=task,
+        workspace_dir=workspace_dir,
+        repo_dir=repo_dir,
+        agent_logs_dir=agent_logs_dir,
+    )
+    verifier_cmd = build_verifier_exec_command(
+        task=task,
+        image_path=image_path,
+        workspace_dir=workspace_dir,
+        task_dir=task.task_dir,
+        verifier_logs_dir=verifier_logs_dir,
+        env=verifier_env,
+    )
+
+    agent_timeout = (task.agent_timeout_sec or 1800.0) * timeout_multiplier
+    verifier_timeout = task.verifier_timeout_sec * timeout_multiplier
+
+    try:
+        agent_proc = run_shell(agent_cmd, timeout=agent_timeout, env={**os.environ, **env})
+        result.agent_return_code = agent_proc.returncode
+        if agent_proc.returncode != 0:
+            result.status = "agent_failed"
+            stdout_path = agent_logs_dir / "stdout.txt"
+            stderr_path = agent_logs_dir / "stderr.txt"
+            stdout_text = stdout_path.read_text(encoding="utf-8", errors="ignore") if stdout_path.exists() else agent_proc.stdout
+            stderr_text = stderr_path.read_text(encoding="utf-8", errors="ignore") if stderr_path.exists() else agent_proc.stderr
+            result.error = (stdout_text + "\n" + stderr_text).strip()[:4000]
+            result.duration_sec = time.time() - start
+            return result
+
+        verifier_proc = run_shell(verifier_cmd, timeout=verifier_timeout)
+        result.verifier_return_code = verifier_proc.returncode
+        reward, reward_metadata = read_reward(verifier_logs_dir)
+        result.reward = reward
+        result.metadata.update(reward_metadata)
+        result.passed = verifier_proc.returncode == 0 and reward is not None and reward > 0.0
+        result.status = "passed" if result.passed else "failed"
+        if not result.passed:
+            stdout_text = (verifier_logs_dir / "test-stdout.txt").read_text(encoding="utf-8", errors="ignore") if (verifier_logs_dir / "test-stdout.txt").exists() else ""
+            stderr_text = (verifier_logs_dir / "test-stderr.txt").read_text(encoding="utf-8", errors="ignore") if (verifier_logs_dir / "test-stderr.txt").exists() else ""
+            result.error = (stdout_text + "\n" + stderr_text).strip()[:4000]
+        return result
+    except subprocess.TimeoutExpired as exc:
+        result.status = "timeout"
+        result.error = f"timeout after {exc.timeout}s"
+        return result
+    except Exception as exc:
+        result.status = "error"
+        result.error = str(exc)
+        return result
+    finally:
+        result.duration_sec = time.time() - start
+
+
+def print_report(results: list[LocalTrialResult]) -> None:
+    total = len(results)
+    passed = sum(1 for item in results if item.passed)
+    failed = sum(1 for item in results if item.status == "failed")
+    skipped = sum(1 for item in results if item.status == "skipped")
+    print()
+    print("=" * 80)
+    print("  TERMINAL-BENCH LOCAL REPORT")
+    print("=" * 80)
+    for item in results:
+        icon = "PASS ✅" if item.passed else ("SKIP ⚪" if item.status == "skipped" else "FAIL ❌")
+        print(f"  {icon:<8} {item.short_name:<32} {item.duration_sec:>7.1f}s  {item.status}")
+    print("─" * 80)
+    print(f"  Total: {total}  Passed: {passed}  Failed: {failed}  Skipped: {skipped}")
+    score = (100.0 * passed / total) if total else 0.0
+    print(f"  Score: {score:.1f}%")
+    print("─" * 80)
+
+
+def save_results(path: Path, results: list[LocalTrialResult]) -> None:
+    payload = {
+        "benchmark": "terminal-bench-local-apptainer",
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
+        "model": os.environ.get("OPENAI_MODEL", "unknown"),
+        "results": [asdict(item) for item in results],
+    }
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Run Terminal-Bench tasks locally with Apptainer and claw-code-agent.")
+    parser.add_argument("--tasks-dir", type=Path, default=DEFAULT_TASKS_DIR, help="Directory containing downloaded Harbor task packages.")
+    parser.add_argument("--include-task-name", "-i", action="append", default=[], help="Include only task names matching this glob pattern.")
+    parser.add_argument("--exclude-task-name", "-x", action="append", default=[], help="Exclude task names matching this glob pattern.")
+    parser.add_argument("--n-tasks", "-l", type=int, default=None, help="Maximum number of tasks to run after filtering.")
+    parser.add_argument("--jobs-dir", "-o", type=Path, default=DEFAULT_JOBS_DIR, help="Directory where trial outputs will be written.")
+    parser.add_argument("--images-dir", type=Path, default=DEFAULT_JOBS_DIR / "_images", help="Directory where pulled Apptainer images will be cached.")
+    parser.add_argument("--force-pull", action="store_true", help="Re-pull Apptainer images even if cached locally.")
+    parser.add_argument("--keep-images", action="store_true", help="Keep pulled SIF images in the image cache directory.")
+    parser.add_argument("--timeout-multiplier", type=float, default=1.0, help="Multiplier applied to agent and verifier timeouts.")
+    parser.add_argument("--output", type=Path, help="Optional JSON file for the run summary.")
+    parser.add_argument("--list", action="store_true", help="List discovered tasks and exit.")
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if not args.tasks_dir.exists():
+        parser.error(f"tasks directory not found: {args.tasks_dir}")
+
+    tasks = discover_tasks(args.tasks_dir)
+    tasks = filter_tasks(
+        tasks,
+        include_patterns=args.include_task_name,
+        exclude_patterns=args.exclude_task_name,
+        limit=args.n_tasks,
+    )
+
+    if args.list:
+        for task in tasks:
+            print(task.short_name)
+        return
+
+    if not tasks:
+        parser.error("no tasks matched")
+
+    repo_dir = Path(__file__).resolve().parent.parent
+    results: list[LocalTrialResult] = []
+    print()
+    print("=" * 80)
+    print("  TERMINAL-BENCH LOCAL")
+    print("=" * 80)
+    print(f"  Tasks:    {len(tasks)}")
+    print(f"  Jobs dir: {args.jobs_dir}")
+    print("=" * 80)
+    print()
+
+    for index, task in enumerate(tasks, 1):
+        print(f"[{index}/{len(tasks)}] {task.short_name}")
+        result = run_trial(
+            task,
+            repo_dir=repo_dir,
+            jobs_dir=args.jobs_dir,
+            images_dir=args.images_dir,
+            force_pull=args.force_pull,
+            keep_images=args.keep_images,
+            timeout_multiplier=args.timeout_multiplier,
+        )
+        results.append(result)
+        icon = "PASS ✅" if result.passed else ("SKIP ⚪" if result.status == "skipped" else "FAIL ❌")
+        print(f"  -> {icon}  ({result.duration_sec:.1f}s)")
+        print(f"     trial_dir: {result.trial_dir}")
+        if result.error:
+            print(f"     error: {result.error[:200]}")
+        print()
+
+    print_report(results)
+    if args.output:
+        save_results(args.output, results)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/suites/aime.py

@@ -110,6 +110,22 @@ class AIMEBenchmark(BenchmarkSuite):
             f"no explanation, no work, just the number."
         )
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        answer_path = Path(workspace) / "answer.txt"
+        if answer_path.exists():
+            return
+        numbers = re.findall(r"-?\d+", agent_output.replace(",", ""))
+        if numbers:
+            answer_path.write_text(numbers[-1] + "\n", encoding="utf-8")
+            metadata["recovered_answer_from_output"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", "unknown")
         expected = str(problem["answer"]).strip()

benchmarks/suites/base.py

@@ -1,17 +1,12 @@
 """
-Base class for all benchmark suites.
-
-Every suite implements:
-  - load_dataset()   -> download/load the evaluation dataset
-  - run_single()     -> run the agent on one problem
-  - evaluate()       -> score agent output for one problem
-  - run_all()        -> orchestrate the full benchmark
+Base class for benchmark suites and shared benchmark helpers.
 """
 
 from __future__ import annotations
 
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -23,10 +18,33 @@ from pathlib import Path
 from typing import Any
 
 
+_SAFE_COMPONENT_RE = re.compile(r"[^A-Za-z0-9_.-]+")
+
+
+def _safe_component(value: str) -> str:
+    cleaned = _SAFE_COMPONENT_RE.sub("_", value).strip("._")
+    return cleaned or "item"
+
+
+def resolve_temp_root() -> Path:
+    root = Path(tempfile.gettempdir()).expanduser().resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def make_temp_workspace(prefix: str, suite_name: str, problem_id: str) -> str:
+    temp_root = resolve_temp_root()
+    safe_prefix = _safe_component(prefix)
+    safe_suite = _safe_component(suite_name)
+    safe_problem = _safe_component(problem_id)
+    return tempfile.mkdtemp(
+        prefix=f"{safe_prefix}_{safe_suite}_{safe_problem}_",
+        dir=str(temp_root),
+    )
+
+
 @dataclass
 class BenchmarkResult:
-    """Result for a single problem in a benchmark suite."""
-
     problem_id: str
     passed: bool
     expected: str = ""
@@ -38,8 +56,6 @@ class BenchmarkResult:
 
 @dataclass
 class SuiteReport:
-    """Aggregated report for a full benchmark suite run."""
-
     suite_name: str
     total: int
     passed: int
@@ -65,12 +81,9 @@ class SuiteReport:
 
 
 class BenchmarkSuite(ABC):
-    """Abstract base for a benchmark suite."""
-
-    # Subclasses set these
     name: str = "base"
     description: str = ""
-    category: str = "general"  # coding | math | instruction-following
+    category: str = "general"
 
     def __init__(
         self,
@@ -79,6 +92,8 @@ class BenchmarkSuite(ABC):
         limit: int | None = None,
         agent_timeout: float = 300.0,
         verbose: bool = False,
+        artifacts_dir: str | None = None,
+        save_passing_artifacts: bool = False,
     ) -> None:
         self.data_dir = data_dir or str(
             Path(__file__).resolve().parent.parent / "data"
@@ -86,37 +101,28 @@ class BenchmarkSuite(ABC):
         self.limit = limit
         self.agent_timeout = agent_timeout
         self.verbose = verbose
+        self.artifacts_dir = artifacts_dir
+        self.save_passing_artifacts = save_passing_artifacts
         self.project_root = str(Path(__file__).resolve().parent.parent.parent)
 
-    # ------------------------------------------------------------------
-    # Abstract interface
-    # ------------------------------------------------------------------
-
     @abstractmethod
     def load_dataset(self) -> list[dict[str, Any]]:
-        """Return a list of problem dicts. Each must have at least an 'id' key."""
         ...
 
     @abstractmethod
     def build_prompt(self, problem: dict[str, Any]) -> str:
-        """Convert a problem dict into the instruction string sent to the agent."""
         ...
 
     @abstractmethod
-    def evaluate(
-        self, problem: dict[str, Any], workspace: str
-    ) -> BenchmarkResult:
-        """Score the agent's output for one problem.  Return a BenchmarkResult."""
+    def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         ...
 
-    # ------------------------------------------------------------------
-    # Agent execution helpers
-    # ------------------------------------------------------------------
-
     def _run_shell(
-        self, cmd: str, cwd: str, timeout: float = 30.0
+        self,
+        cmd: str,
+        cwd: str,
+        timeout: float = 30.0,
     ) -> tuple[int, str]:
-        """Run a shell command, return (exit_code, combined_output)."""
         try:
             proc = subprocess.run(
                 cmd,
@@ -133,10 +139,6 @@ class BenchmarkSuite(ABC):
             return 1, str(exc)
 
     def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
-        """Run the claw-code-agent on *instruction* inside *workspace*.
-
-        Returns (exit_code, output, duration_sec).
-        """
         import shlex
 
         agent_cmd = (
@@ -147,11 +149,13 @@ class BenchmarkSuite(ABC):
             f"--allow-shell"
         )
         if self.verbose:
-            print(f"  agent cmd: {agent_cmd[:120]}...")
+            print(f"  agent cmd: {agent_cmd[:160]}...")
 
         start = time.time()
         code, output = self._run_shell(
-            agent_cmd, cwd=self.project_root, timeout=self.agent_timeout
+            agent_cmd,
+            cwd=self.project_root,
+            timeout=self.agent_timeout,
         )
         duration = time.time() - start
 
@@ -160,12 +164,65 @@ class BenchmarkSuite(ABC):
 
         return code, output, duration
 
-    # ------------------------------------------------------------------
-    # Orchestration
-    # ------------------------------------------------------------------
+    def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
+        del problem, workspace
+
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem, workspace, agent_output, metadata
+
+    def _artifact_root(self, index: int, problem_id: str) -> Path | None:
+        if not self.artifacts_dir:
+            return None
+        root = Path(self.artifacts_dir)
+        root.mkdir(parents=True, exist_ok=True)
+        return root / f"{index:03d}_{_safe_component(problem_id)}"
+
+    def _save_artifacts(
+        self,
+        *,
+        index: int,
+        problem: dict[str, Any],
+        prompt: str,
+        agent_output: str,
+        workspace: str,
+        result: BenchmarkResult,
+        agent_exit_code: int,
+    ) -> None:
+        artifact_root = self._artifact_root(index, result.problem_id)
+        if artifact_root is None:
+            return
+        if result.passed and not self.save_passing_artifacts:
+            return
+
+        artifact_root.mkdir(parents=True, exist_ok=True)
+        (artifact_root / "problem.json").write_text(
+            json.dumps(problem, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        (artifact_root / "prompt.txt").write_text(prompt, encoding="utf-8")
+        (artifact_root / "agent_output.txt").write_text(agent_output, encoding="utf-8")
+
+        workspace_dst = artifact_root / "workspace"
+        if workspace_dst.exists():
+            shutil.rmtree(workspace_dst, ignore_errors=True)
+        shutil.copytree(workspace, workspace_dst)
+
+        result_payload = asdict(result)
+        result_payload["agent_exit_code"] = agent_exit_code
+        result_payload["workspace"] = workspace
+        (artifact_root / "result.json").write_text(
+            json.dumps(result_payload, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        result.metadata["artifact_path"] = str(artifact_root)
 
     def run_all(self) -> SuiteReport:
-        """Run the full benchmark suite and return a SuiteReport."""
         problems = self.load_dataset()
         if self.limit is not None:
             problems = problems[: self.limit]
@@ -182,25 +239,37 @@ class BenchmarkSuite(ABC):
         print("=" * 72)
         print()
 
-        all_results: list[BenchmarkResult] = []
         suite_start = time.time()
+        all_results: list[BenchmarkResult] = []
 
-        for i, problem in enumerate(problems, 1):
-            pid = problem.get("id", problem.get("task_id", f"problem-{i}"))
-            print(f"[{i}/{len(problems)}] {pid}")
+        for index, problem in enumerate(problems, 1):
+            pid = str(problem.get("id", problem.get("task_id", f"problem-{index}")))
+            print(f"[{index}/{len(problems)}] {pid}")
 
-            workspace = tempfile.mkdtemp(prefix=f"claw_{self.name}_{pid}_")
+            workspace = make_temp_workspace("claw", self.name, pid)
+            prompt = ""
+            agent_output = ""
+            agent_exit_code = -1
             try:
-                # Prepare workspace
                 self.setup_workspace(problem, workspace)
-
-                # Build prompt and run agent
                 prompt = self.build_prompt(problem)
-                _code, _output, duration = self.run_agent(prompt, workspace)
+                agent_exit_code, agent_output, duration = self.run_agent(prompt, workspace)
 
-                # Evaluate
+                result_metadata: dict[str, Any] = {"agent_exit_code": agent_exit_code}
+                self.recover_output_files(problem, workspace, agent_output, result_metadata)
                 result = self.evaluate(problem, workspace)
                 result.duration_sec = duration
+                result.metadata.update(result_metadata)
+
+                self._save_artifacts(
+                    index=index,
+                    problem=problem,
+                    prompt=prompt,
+                    agent_output=agent_output,
+                    workspace=workspace,
+                    result=result,
+                    agent_exit_code=agent_exit_code,
+                )
 
                 status = "PASS ✅" if result.passed else "FAIL ❌"
                 print(f"  -> {status}  ({duration:.1f}s)")
@@ -209,6 +278,16 @@ class BenchmarkSuite(ABC):
                     problem_id=pid,
                     passed=False,
                     error=str(exc),
+                    metadata={"agent_exit_code": agent_exit_code},
+                )
+                self._save_artifacts(
+                    index=index,
+                    problem=problem,
+                    prompt=prompt,
+                    agent_output=agent_output,
+                    workspace=workspace,
+                    result=result,
+                    agent_exit_code=agent_exit_code,
                 )
                 print(f"  -> ERROR ❌  {exc}")
             finally:
@@ -218,9 +297,8 @@ class BenchmarkSuite(ABC):
             print()
 
         suite_duration = time.time() - suite_start
-        passed = sum(1 for r in all_results if r.passed)
+        passed = sum(1 for item in all_results if item.passed)
         total = len(all_results)
-
         report = SuiteReport(
             suite_name=self.name,
             total=total,
@@ -232,19 +310,9 @@ class BenchmarkSuite(ABC):
             results=all_results,
             timestamp=time.strftime("%Y-%m-%dT%H:%M:%S"),
         )
-
         self._print_report(report)
         return report
 
-    def setup_workspace(
-        self, problem: dict[str, Any], workspace: str
-    ) -> None:
-        """Optional: prepare files in workspace before the agent runs."""
-
-    # ------------------------------------------------------------------
-    # Reporting
-    # ------------------------------------------------------------------
-
     @staticmethod
     def _print_report(report: SuiteReport) -> None:
         print()
@@ -252,9 +320,9 @@ class BenchmarkSuite(ABC):
         print(f"  {report.suite_name} — RESULTS")
         print("=" * 72)
         print()
-        for r in report.results:
-            icon = "✅" if r.passed else "❌"
-            print(f"  {icon} {r.problem_id:<40} {r.duration_sec:.1f}s")
+        for result in report.results:
+            icon = "✅" if result.passed else "❌"
+            print(f"  {icon} {result.problem_id:<40} {result.duration_sec:.1f}s")
         print()
         print("─" * 72)
         print(
@@ -267,7 +335,6 @@ class BenchmarkSuite(ABC):
 
     @staticmethod
     def save_report(report: SuiteReport, path: str) -> None:
-        """Persist a SuiteReport to JSON."""
-        with open(path, "w") as fh:
+        with open(path, "w", encoding="utf-8") as fh:
             json.dump(report.to_dict(), fh, indent=2)
         print(f"  Report saved to {path}")

benchmarks/suites/gsm8k.py

@@ -141,6 +141,22 @@ class GSM8KBenchmark(BenchmarkSuite):
             f"Save just the number to answer.txt — no units, no explanation."
         )
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        answer_path = Path(workspace) / "answer.txt"
+        if answer_path.exists():
+            return
+        number = _extract_number(agent_output)
+        if number is not None:
+            answer_path.write_text(number + "\n", encoding="utf-8")
+            metadata["recovered_answer_from_output"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", "unknown")
         expected = str(problem["answer"])

benchmarks/suites/humaneval.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import subprocess
 import sys
 import textwrap
@@ -213,19 +214,36 @@ class HumanEvalBenchmark(BenchmarkSuite):
         # Write the test harness so we can verify after the agent runs
         test_code = problem["test"]
         entry = problem["entry_point"]
-        harness = textwrap.dedent(f"""\
-            import sys
-            sys.path.insert(0, ".")
-            from solution import {entry}
-
-            {test_code}
-
-            check({entry})
-            print("ALL_TESTS_PASSED")
-        """)
+        harness = (
+            "import sys\n"
+            'sys.path.insert(0, ".")\n'
+            f"from solution import {entry}\n\n"
+            f"{test_code}\n\n"
+            f"check({entry})\n"
+            'print("ALL_TESTS_PASSED")\n'
+        )
         with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
             fh.write(harness)
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        solution_path = Path(workspace) / "solution.py"
+        if solution_path.exists():
+            return
+        blocks = re.findall(r"```(?:python)?\s*(.*?)```", agent_output, flags=re.DOTALL | re.IGNORECASE)
+        for block in blocks:
+            candidate = block.strip()
+            if "def " in candidate or "import " in candidate or "from " in candidate:
+                solution_path.write_text(candidate.rstrip() + "\n", encoding="utf-8")
+                metadata["recovered_solution_from_output"] = True
+                return
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("task_id", problem.get("id", "unknown"))
         sol = os.path.join(workspace, "solution.py")

benchmarks/suites/ifeval.py

@@ -215,6 +215,22 @@ class IFEvalBenchmark(BenchmarkSuite):
     def build_prompt(self, problem: dict[str, Any]) -> str:
         return problem["instruction"]
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        output_path = Path(workspace) / "output.txt"
+        if output_path.exists():
+            return
+        text = agent_output.strip()
+        if text:
+            output_path.write_text(text + "\n", encoding="utf-8")
+            metadata["recovered_output_from_agent_text"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", "unknown")
         checks = problem.get("checks", [])

benchmarks/suites/math_bench.py

@@ -182,6 +182,22 @@ class MATHBenchmark(BenchmarkSuite):
             f"Save just the answer to answer.txt — no explanation, no work, just the answer."
         )
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        answer_path = Path(workspace) / "answer.txt"
+        if answer_path.exists():
+            return
+        answer = _normalize_answer(agent_output)
+        if answer:
+            answer_path.write_text(answer + "\n", encoding="utf-8")
+            metadata["recovered_answer_from_output"] = True
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = problem.get("id", str(problem.get("task_id", "unknown")))
         expected_raw = str(problem["answer"])

benchmarks/suites/mbpp.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import sys
 import textwrap
 from pathlib import Path
@@ -213,18 +214,35 @@ class MBPPBenchmark(BenchmarkSuite):
     def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
         tests = problem.get("test_list", [])
         test_code = "\n".join(tests)
-        harness = textwrap.dedent(f"""\
-            import sys
-            sys.path.insert(0, ".")
-            from solution import *
-
-            {test_code}
-
-            print("ALL_TESTS_PASSED")
-        """)
+        harness = (
+            "import sys\n"
+            'sys.path.insert(0, ".")\n'
+            "from solution import *\n\n"
+            f"{test_code}\n\n"
+            'print("ALL_TESTS_PASSED")\n'
+        )
         with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
             fh.write(harness)
 
+    def recover_output_files(
+        self,
+        problem: dict[str, Any],
+        workspace: str,
+        agent_output: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        del problem
+        solution_path = Path(workspace) / "solution.py"
+        if solution_path.exists():
+            return
+        blocks = re.findall(r"```(?:python)?\s*(.*?)```", agent_output, flags=re.DOTALL | re.IGNORECASE)
+        for block in blocks:
+            candidate = block.strip()
+            if "def " in candidate or "import " in candidate or "class " in candidate:
+                solution_path.write_text(candidate.rstrip() + "\n", encoding="utf-8")
+                metadata["recovered_solution_from_output"] = True
+                return
+
     def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
         pid = str(problem.get("task_id", "unknown"))
         sol = os.path.join(workspace, "solution.py")

harbor_adapter.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import os
+import shlex
+
+from harbor.agents.installed.base import BaseInstalledAgent, CliFlag, EnvVar, with_prompt_template
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+
+
+class ClawCodeInstalledAgent(BaseInstalledAgent):
+    """
+    Harbor installed-agent adapter for claw-code-agent.
+
+    This installs the published GitHub repo inside the Harbor environment and
+    runs the CLI in one-shot mode against the task working directory.
+    """
+
+    CLI_FLAGS = [
+        CliFlag("max_turns", cli="--max-turns", type="int"),
+        CliFlag("append_system_prompt", cli="--append-system-prompt", type="str"),
+        CliFlag("system_prompt", cli="--system-prompt", type="str"),
+        CliFlag("stream", cli="--stream", type="bool"),
+        CliFlag("allow_write", cli="--allow-write", type="bool", default=True),
+        CliFlag("allow_shell", cli="--allow-shell", type="bool", default=True),
+    ]
+
+    ENV_VARS = [
+        EnvVar("openai_api_key", env="OPENAI_API_KEY", type="str", env_fallback="OPENAI_API_KEY"),
+        EnvVar("openai_base_url", env="OPENAI_BASE_URL", type="str", env_fallback="OPENAI_BASE_URL"),
+        EnvVar("openai_model", env="OPENAI_MODEL", type="str", env_fallback="OPENAI_MODEL"),
+    ]
+
+    @staticmethod
+    def name() -> str:
+        return "claw-code-agent"
+
+    def get_version_command(self) -> str | None:
+        return "claw-code-agent --help >/dev/null 2>&1 && echo installed"
+
+    async def install(self, environment: BaseEnvironment) -> None:
+        await self.exec_as_root(
+            environment,
+            command=(
+                "if command -v apk >/dev/null 2>&1; then "
+                "apk add --no-cache python3 py3-pip bash curl git; "
+                "elif command -v apt-get >/dev/null 2>&1; then "
+                "apt-get update && apt-get install -y python3 python3-pip bash curl git; "
+                "elif command -v yum >/dev/null 2>&1; then "
+                "yum install -y python3 python3-pip bash curl git; "
+                "fi"
+            ),
+            env={"DEBIAN_FRONTEND": "noninteractive"},
+        )
+        version_spec = f"@{self._version}" if self._version else ""
+        repo_spec = (
+            f"git+https://github.com/HarnessLab/claw-code-agent.git{version_spec}"
+            if version_spec
+            else "git+https://github.com/HarnessLab/claw-code-agent.git"
+        )
+        await self.exec_as_agent(
+            environment,
+            command=(
+                "set -euo pipefail; "
+                f"python3 -m pip install --upgrade pip && python3 -m pip install {shlex.quote(repo_spec)} && "
+                "claw-code-agent --help >/dev/null"
+            ),
+        )
+
+    def populate_context_post_run(self, context: AgentContext) -> None:
+        del context
+
+    @with_prompt_template
+    async def run(
+        self,
+        instruction: str,
+        environment: BaseEnvironment,
+        context: AgentContext,
+    ) -> None:
+        del context
+        env = self.resolve_env_vars()
+        if self.model_name and "OPENAI_MODEL" not in env:
+            env["OPENAI_MODEL"] = self.model_name
+
+        cli_flags = self.build_cli_flags()
+        extra_flags = (cli_flags + " ") if cli_flags else ""
+        escaped_instruction = shlex.quote(instruction)
+
+        # Harbor executes in the task working directory; keep cwd anchored there.
+        command = (
+            f"claw-code-agent agent {escaped_instruction} "
+            f"--cwd $(pwd) {extra_flags}"
+            "2>&1 | stdbuf -oL tee /logs/agent/claw-code-agent.txt"
+        )
+        await self.exec_as_agent(environment, command=command, env=env)

install_dockor.sh

@@ -0,0 +1,36 @@
+# everything stays under $SCRATCH
+export DROOT="$SCRATCH/docker-rootless"
+mkdir -p "$DROOT"/{bin,home,config/docker,run,data}
+chmod 700 "$DROOT/run"
+
+# read-only checks; these do not modify the system
+command -v newuidmap newgidmap
+grep "^$(whoami):" /etc/subuid /etc/subgid
+cat /proc/sys/kernel/unprivileged_userns_clone 2>/dev/null || true
+cat /proc/sys/user/max_user_namespaces 2>/dev/null || true
+df -T "$SCRATCH"
+
+# keep Docker files in $SCRATCH
+export HOME="$DROOT/home"
+export DOCKER_BIN="$DROOT/bin"
+export XDG_CONFIG_HOME="$DROOT/config"
+export XDG_RUNTIME_DIR="$DROOT/run"
+export PATH="$DOCKER_BIN:$PATH"
+export DOCKER_HOST="unix://$XDG_RUNTIME_DIR/docker.sock"
+
+# install rootless Docker binaries into $SCRATCH
+curl -fsSL https://get.docker.com/rootless | sh
+
+# store Docker images/layers in $SCRATCH too
+cat > "$XDG_CONFIG_HOME/docker/daemon.json" <<EOF
+{
+  "data-root": "$DROOT/data"
+}
+EOF
+
+# start daemon manually (no systemd)
+nohup dockerd-rootless.sh > "$DROOT/dockerd.log" 2>&1 &
+sleep 5
+
+docker --version
+docker info

pyproject.toml

@@ -51,5 +51,4 @@ include = ["src*"]
 [tool.setuptools.package-data]
 src = [
   "reference_data/*.json",
-  "reference_data/subsystems/*.json",
 ]

src/QueryEngine.py

@@ -1,19 +0,0 @@
-from __future__ import annotations
-
-from .query_engine import QueryEnginePort
-from .runtime import PortRuntime
-
-
-class QueryEngineRuntime(QueryEnginePort):
-    def route(self, prompt: str, limit: int = 5) -> str:
-        matches = PortRuntime().route_prompt(prompt, limit=limit)
-        lines = ['# Query Engine Route', '', f'Prompt: {prompt}', '']
-        if not matches:
-            lines.append('No mirrored command/tool matches found.')
-            return '\n'.join(lines)
-        lines.append('Matches:')
-        lines.extend(f'- [{match.kind}] {match.name} ({match.score}) — {match.source_hint}' for match in matches)
-        return '\n'.join(lines)
-
-
-__all__ = ['QueryEnginePort', 'QueryEngineRuntime']

src/Tool.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class ToolDefinition:
-    name: str
-    purpose: str
-
-
-DEFAULT_TOOLS = (
-    ToolDefinition('port_manifest', 'Summarize the active Python workspace'),
-    ToolDefinition('query_engine', 'Render a Python-first porting summary'),
-)

src/assistant/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `assistant` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'assistant.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/bootstrap/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `bootstrap` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'bootstrap.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/bridge/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `bridge` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'bridge.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/buddy/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `buddy` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'buddy.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/cli/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `cli` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'cli.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/components/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `components` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'components.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/constants/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `constants` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'constants.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/coordinator/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `coordinator` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'coordinator.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/costHook.py

@@ -1,8 +0,0 @@
-from __future__ import annotations
-
-from .cost_tracker import CostTracker
-
-
-def apply_cost_hook(tracker: CostTracker, label: str, units: int) -> CostTracker:
-    tracker.record(label, units)
-    return tracker

src/dialogLaunchers.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class DialogLauncher:
-    name: str
-    description: str
-
-
-DEFAULT_DIALOGS = (
-    DialogLauncher('summary', 'Launch the Markdown summary view'),
-    DialogLauncher('parity_audit', 'Launch the parity audit view'),
-)

src/direct_modes.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class DirectModeReport:
-    mode: str
-    target: str
-    active: bool
-
-    def as_text(self) -> str:
-        return f'mode={self.mode}\ntarget={self.target}\nactive={self.active}'
-
-
-def run_direct_connect(target: str) -> DirectModeReport:
-    return DirectModeReport(mode='direct-connect', target=target, active=True)
-
-
-def run_deep_link(target: str) -> DirectModeReport:
-    return DirectModeReport(mode='deep-link', target=target, active=True)

src/entrypoints/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `entrypoints` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'entrypoints.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/hooks/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `hooks` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'hooks.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/ink.py

@@ -1,6 +0,0 @@
-from __future__ import annotations
-
-
-def render_markdown_panel(text: str) -> str:
-    border = '=' * 40
-    return f"{border}\n{text}\n{border}"

src/interactiveHelpers.py

@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-
-def bulletize(items: list[str]) -> str:
-    return '\n'.join(f'- {item}' for item in items)

src/keybindings/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `keybindings` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'keybindings.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/memdir/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `memdir` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'memdir.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/migrations/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `migrations` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'migrations.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/moreright/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `moreright` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'moreright.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/native_ts/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `native-ts` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'native_ts.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/outputStyles/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `outputStyles` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'outputStyles.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/plugins/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `plugins` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'plugins.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/projectOnboardingState.py

@@ -1,10 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-@dataclass
-class ProjectOnboardingState:
-    has_readme: bool
-    has_tests: bool
-    python_first: bool = True

src/query.py

@@ -1,13 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class QueryRequest:
-    prompt: str
-
-
-@dataclass(frozen=True)
-class QueryResponse:
-    text: str

src/reference_data/subsystems/assistant.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "assistant",
-  "package_name": "assistant",
-  "module_count": 1,
-  "sample_files": [
-    "assistant/sessionHistory.ts"
-  ]
-}

src/reference_data/subsystems/bootstrap.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "bootstrap",
-  "package_name": "bootstrap",
-  "module_count": 1,
-  "sample_files": [
-    "bootstrap/state.ts"
-  ]
-}

src/reference_data/subsystems/bridge.json

@@ -1,32 +0,0 @@
-{
-  "archive_name": "bridge",
-  "package_name": "bridge",
-  "module_count": 31,
-  "sample_files": [
-    "bridge/bridgeApi.ts",
-    "bridge/bridgeConfig.ts",
-    "bridge/bridgeDebug.ts",
-    "bridge/bridgeEnabled.ts",
-    "bridge/bridgeMain.ts",
-    "bridge/bridgeMessaging.ts",
-    "bridge/bridgePermissionCallbacks.ts",
-    "bridge/bridgePointer.ts",
-    "bridge/bridgeStatusUtil.ts",
-    "bridge/bridgeUI.ts",
-    "bridge/capacityWake.ts",
-    "bridge/codeSessionApi.ts",
-    "bridge/createSession.ts",
-    "bridge/debugUtils.ts",
-    "bridge/envLessBridgeConfig.ts",
-    "bridge/flushGate.ts",
-    "bridge/inboundAttachments.ts",
-    "bridge/inboundMessages.ts",
-    "bridge/initReplBridge.ts",
-    "bridge/jwtUtils.ts",
-    "bridge/pollConfig.ts",
-    "bridge/pollConfigDefaults.ts",
-    "bridge/remoteBridgeCore.ts",
-    "bridge/replBridge.ts",
-    "bridge/replBridgeHandle.ts"
-  ]
-}

src/reference_data/subsystems/buddy.json

@@ -1,13 +0,0 @@
-{
-  "archive_name": "buddy",
-  "package_name": "buddy",
-  "module_count": 6,
-  "sample_files": [
-    "buddy/CompanionSprite.tsx",
-    "buddy/companion.ts",
-    "buddy/prompt.ts",
-    "buddy/sprites.ts",
-    "buddy/types.ts",
-    "buddy/useBuddyNotification.tsx"
-  ]
-}

src/reference_data/subsystems/cli.json

@@ -1,26 +0,0 @@
-{
-  "archive_name": "cli",
-  "package_name": "cli",
-  "module_count": 19,
-  "sample_files": [
-    "cli/exit.ts",
-    "cli/handlers/agents.ts",
-    "cli/handlers/auth.ts",
-    "cli/handlers/autoMode.ts",
-    "cli/handlers/mcp.tsx",
-    "cli/handlers/plugins.ts",
-    "cli/handlers/util.tsx",
-    "cli/ndjsonSafeStringify.ts",
-    "cli/print.ts",
-    "cli/remoteIO.ts",
-    "cli/structuredIO.ts",
-    "cli/transports/HybridTransport.ts",
-    "cli/transports/SSETransport.ts",
-    "cli/transports/SerialBatchEventUploader.ts",
-    "cli/transports/WebSocketTransport.ts",
-    "cli/transports/WorkerStateUploader.ts",
-    "cli/transports/ccrClient.ts",
-    "cli/transports/transportUtils.ts",
-    "cli/update.ts"
-  ]
-}

src/reference_data/subsystems/components.json

@@ -1,32 +0,0 @@
-{
-  "archive_name": "components",
-  "package_name": "components",
-  "module_count": 389,
-  "sample_files": [
-    "components/AgentProgressLine.tsx",
-    "components/App.tsx",
-    "components/ApproveApiKey.tsx",
-    "components/AutoModeOptInDialog.tsx",
-    "components/AutoUpdater.tsx",
-    "components/AutoUpdaterWrapper.tsx",
-    "components/AwsAuthStatusBox.tsx",
-    "components/BaseTextInput.tsx",
-    "components/BashModeProgress.tsx",
-    "components/BridgeDialog.tsx",
-    "components/BypassPermissionsModeDialog.tsx",
-    "components/ChannelDowngradeDialog.tsx",
-    "components/ClaudeCodeHint/PluginHintMenu.tsx",
-    "components/ClaudeInChromeOnboarding.tsx",
-    "components/ClaudeMdExternalIncludesDialog.tsx",
-    "components/ClickableImageRef.tsx",
-    "components/CompactSummary.tsx",
-    "components/ConfigurableShortcutHint.tsx",
-    "components/ConsoleOAuthFlow.tsx",
-    "components/ContextSuggestions.tsx",
-    "components/ContextVisualization.tsx",
-    "components/CoordinatorAgentStatus.tsx",
-    "components/CostThresholdDialog.tsx",
-    "components/CtrlOToExpand.tsx",
-    "components/CustomSelect/SelectMulti.tsx"
-  ]
-}

src/reference_data/subsystems/constants.json

@@ -1,28 +0,0 @@
-{
-  "archive_name": "constants",
-  "package_name": "constants",
-  "module_count": 21,
-  "sample_files": [
-    "constants/apiLimits.ts",
-    "constants/betas.ts",
-    "constants/common.ts",
-    "constants/cyberRiskInstruction.ts",
-    "constants/errorIds.ts",
-    "constants/figures.ts",
-    "constants/files.ts",
-    "constants/github-app.ts",
-    "constants/keys.ts",
-    "constants/messages.ts",
-    "constants/oauth.ts",
-    "constants/outputStyles.ts",
-    "constants/product.ts",
-    "constants/prompts.ts",
-    "constants/spinnerVerbs.ts",
-    "constants/system.ts",
-    "constants/systemPromptSections.ts",
-    "constants/toolLimits.ts",
-    "constants/tools.ts",
-    "constants/turnCompletionVerbs.ts",
-    "constants/xml.ts"
-  ]
-}

src/reference_data/subsystems/coordinator.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "coordinator",
-  "package_name": "coordinator",
-  "module_count": 1,
-  "sample_files": [
-    "coordinator/coordinatorMode.ts"
-  ]
-}

src/reference_data/subsystems/entrypoints.json

@@ -1,15 +0,0 @@
-{
-  "archive_name": "entrypoints",
-  "package_name": "entrypoints",
-  "module_count": 8,
-  "sample_files": [
-    "entrypoints/agentSdkTypes.ts",
-    "entrypoints/cli.tsx",
-    "entrypoints/init.ts",
-    "entrypoints/mcp.ts",
-    "entrypoints/sandboxTypes.ts",
-    "entrypoints/sdk/controlSchemas.ts",
-    "entrypoints/sdk/coreSchemas.ts",
-    "entrypoints/sdk/coreTypes.ts"
-  ]
-}

src/reference_data/subsystems/hooks.json

@@ -1,32 +0,0 @@
-{
-  "archive_name": "hooks",
-  "package_name": "hooks",
-  "module_count": 104,
-  "sample_files": [
-    "hooks/fileSuggestions.ts",
-    "hooks/notifs/useAutoModeUnavailableNotification.ts",
-    "hooks/notifs/useCanSwitchToExistingSubscription.tsx",
-    "hooks/notifs/useDeprecationWarningNotification.tsx",
-    "hooks/notifs/useFastModeNotification.tsx",
-    "hooks/notifs/useIDEStatusIndicator.tsx",
-    "hooks/notifs/useInstallMessages.tsx",
-    "hooks/notifs/useLspInitializationNotification.tsx",
-    "hooks/notifs/useMcpConnectivityStatus.tsx",
-    "hooks/notifs/useModelMigrationNotifications.tsx",
-    "hooks/notifs/useNpmDeprecationNotification.tsx",
-    "hooks/notifs/usePluginAutoupdateNotification.tsx",
-    "hooks/notifs/usePluginInstallationStatus.tsx",
-    "hooks/notifs/useRateLimitWarningNotification.tsx",
-    "hooks/notifs/useSettingsErrors.tsx",
-    "hooks/notifs/useStartupNotification.ts",
-    "hooks/notifs/useTeammateShutdownNotification.ts",
-    "hooks/renderPlaceholder.ts",
-    "hooks/toolPermission/PermissionContext.ts",
-    "hooks/toolPermission/handlers/coordinatorHandler.ts",
-    "hooks/toolPermission/handlers/interactiveHandler.ts",
-    "hooks/toolPermission/handlers/swarmWorkerHandler.ts",
-    "hooks/toolPermission/permissionLogging.ts",
-    "hooks/unifiedSuggestions.ts",
-    "hooks/useAfterFirstRender.ts"
-  ]
-}

src/reference_data/subsystems/keybindings.json

@@ -1,21 +0,0 @@
-{
-  "archive_name": "keybindings",
-  "package_name": "keybindings",
-  "module_count": 14,
-  "sample_files": [
-    "keybindings/KeybindingContext.tsx",
-    "keybindings/KeybindingProviderSetup.tsx",
-    "keybindings/defaultBindings.ts",
-    "keybindings/loadUserBindings.ts",
-    "keybindings/match.ts",
-    "keybindings/parser.ts",
-    "keybindings/reservedShortcuts.ts",
-    "keybindings/resolver.ts",
-    "keybindings/schema.ts",
-    "keybindings/shortcutFormat.ts",
-    "keybindings/template.ts",
-    "keybindings/useKeybinding.ts",
-    "keybindings/useShortcutDisplay.ts",
-    "keybindings/validate.ts"
-  ]
-}

src/reference_data/subsystems/memdir.json

@@ -1,15 +0,0 @@
-{
-  "archive_name": "memdir",
-  "package_name": "memdir",
-  "module_count": 8,
-  "sample_files": [
-    "memdir/findRelevantMemories.ts",
-    "memdir/memdir.ts",
-    "memdir/memoryAge.ts",
-    "memdir/memoryScan.ts",
-    "memdir/memoryTypes.ts",
-    "memdir/paths.ts",
-    "memdir/teamMemPaths.ts",
-    "memdir/teamMemPrompts.ts"
-  ]
-}

src/reference_data/subsystems/migrations.json

@@ -1,18 +0,0 @@
-{
-  "archive_name": "migrations",
-  "package_name": "migrations",
-  "module_count": 11,
-  "sample_files": [
-    "migrations/migrateAutoUpdatesToSettings.ts",
-    "migrations/migrateBypassPermissionsAcceptedToSettings.ts",
-    "migrations/migrateEnableAllProjectMcpServersToSettings.ts",
-    "migrations/migrateFennecToOpus.ts",
-    "migrations/migrateLegacyOpusToCurrent.ts",
-    "migrations/migrateOpusToOpus1m.ts",
-    "migrations/migrateReplBridgeEnabledToRemoteControlAtStartup.ts",
-    "migrations/migrateSonnet1mToSonnet45.ts",
-    "migrations/migrateSonnet45ToSonnet46.ts",
-    "migrations/resetAutoModeOptInForDefaultOffer.ts",
-    "migrations/resetProToOpusDefault.ts"
-  ]
-}

src/reference_data/subsystems/moreright.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "moreright",
-  "package_name": "moreright",
-  "module_count": 1,
-  "sample_files": [
-    "moreright/useMoreRight.tsx"
-  ]
-}

src/reference_data/subsystems/native_ts.json

@@ -1,11 +0,0 @@
-{
-  "archive_name": "native-ts",
-  "package_name": "native_ts",
-  "module_count": 4,
-  "sample_files": [
-    "native-ts/color-diff/index.ts",
-    "native-ts/file-index/index.ts",
-    "native-ts/yoga-layout/enums.ts",
-    "native-ts/yoga-layout/index.ts"
-  ]
-}

src/reference_data/subsystems/outputStyles.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "outputStyles",
-  "package_name": "outputStyles",
-  "module_count": 1,
-  "sample_files": [
-    "outputStyles/loadOutputStylesDir.ts"
-  ]
-}

src/reference_data/subsystems/plugins.json

@@ -1,9 +0,0 @@
-{
-  "archive_name": "plugins",
-  "package_name": "plugins",
-  "module_count": 2,
-  "sample_files": [
-    "plugins/builtinPlugins.ts",
-    "plugins/bundled/index.ts"
-  ]
-}

src/reference_data/subsystems/remote.json

@@ -1,11 +0,0 @@
-{
-  "archive_name": "remote",
-  "package_name": "remote",
-  "module_count": 4,
-  "sample_files": [
-    "remote/RemoteSessionManager.ts",
-    "remote/SessionsWebSocket.ts",
-    "remote/remotePermissionBridge.ts",
-    "remote/sdkMessageAdapter.ts"
-  ]
-}

src/reference_data/subsystems/schemas.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "schemas",
-  "package_name": "schemas",
-  "module_count": 1,
-  "sample_files": [
-    "schemas/hooks.ts"
-  ]
-}

src/reference_data/subsystems/screens.json

@@ -1,10 +0,0 @@
-{
-  "archive_name": "screens",
-  "package_name": "screens",
-  "module_count": 3,
-  "sample_files": [
-    "screens/Doctor.tsx",
-    "screens/REPL.tsx",
-    "screens/ResumeConversation.tsx"
-  ]
-}

src/reference_data/subsystems/server.json

@@ -1,10 +0,0 @@
-{
-  "archive_name": "server",
-  "package_name": "server",
-  "module_count": 3,
-  "sample_files": [
-    "server/createDirectConnectSession.ts",
-    "server/directConnectManager.ts",
-    "server/types.ts"
-  ]
-}

src/reference_data/subsystems/services.json

@@ -1,32 +0,0 @@
-{
-  "archive_name": "services",
-  "package_name": "services",
-  "module_count": 130,
-  "sample_files": [
-    "services/AgentSummary/agentSummary.ts",
-    "services/MagicDocs/magicDocs.ts",
-    "services/MagicDocs/prompts.ts",
-    "services/PromptSuggestion/promptSuggestion.ts",
-    "services/PromptSuggestion/speculation.ts",
-    "services/SessionMemory/prompts.ts",
-    "services/SessionMemory/sessionMemory.ts",
-    "services/SessionMemory/sessionMemoryUtils.ts",
-    "services/analytics/config.ts",
-    "services/analytics/datadog.ts",
-    "services/analytics/firstPartyEventLogger.ts",
-    "services/analytics/firstPartyEventLoggingExporter.ts",
-    "services/analytics/growthbook.ts",
-    "services/analytics/index.ts",
-    "services/analytics/metadata.ts",
-    "services/analytics/sink.ts",
-    "services/analytics/sinkKillswitch.ts",
-    "services/api/adminRequests.ts",
-    "services/api/bootstrap.ts",
-    "services/api/claude.ts",
-    "services/api/client.ts",
-    "services/api/dumpPrompts.ts",
-    "services/api/emptyUsage.ts",
-    "services/api/errorUtils.ts",
-    "services/api/errors.ts"
-  ]
-}

src/reference_data/subsystems/skills.json

@@ -1,27 +0,0 @@
-{
-  "archive_name": "skills",
-  "package_name": "skills",
-  "module_count": 20,
-  "sample_files": [
-    "skills/bundled/batch.ts",
-    "skills/bundled/claudeApi.ts",
-    "skills/bundled/claudeApiContent.ts",
-    "skills/bundled/claudeInChrome.ts",
-    "skills/bundled/debug.ts",
-    "skills/bundled/index.ts",
-    "skills/bundled/keybindings.ts",
-    "skills/bundled/loop.ts",
-    "skills/bundled/loremIpsum.ts",
-    "skills/bundled/remember.ts",
-    "skills/bundled/scheduleRemoteAgents.ts",
-    "skills/bundled/simplify.ts",
-    "skills/bundled/skillify.ts",
-    "skills/bundled/stuck.ts",
-    "skills/bundled/updateConfig.ts",
-    "skills/bundled/verify.ts",
-    "skills/bundled/verifyContent.ts",
-    "skills/bundledSkills.ts",
-    "skills/loadSkillsDir.ts",
-    "skills/mcpSkillBuilders.ts"
-  ]
-}

src/reference_data/subsystems/state.json

@@ -1,13 +0,0 @@
-{
-  "archive_name": "state",
-  "package_name": "state",
-  "module_count": 6,
-  "sample_files": [
-    "state/AppState.tsx",
-    "state/AppStateStore.ts",
-    "state/onChangeAppState.ts",
-    "state/selectors.ts",
-    "state/store.ts",
-    "state/teammateViewHelpers.ts"
-  ]
-}

src/reference_data/subsystems/types.json

@@ -1,18 +0,0 @@
-{
-  "archive_name": "types",
-  "package_name": "types",
-  "module_count": 11,
-  "sample_files": [
-    "types/command.ts",
-    "types/generated/events_mono/claude_code/v1/claude_code_internal_event.ts",
-    "types/generated/events_mono/common/v1/auth.ts",
-    "types/generated/events_mono/growthbook/v1/growthbook_experiment_event.ts",
-    "types/generated/google/protobuf/timestamp.ts",
-    "types/hooks.ts",
-    "types/ids.ts",
-    "types/logs.ts",
-    "types/permissions.ts",
-    "types/plugin.ts",
-    "types/textInputTypes.ts"
-  ]
-}

src/reference_data/subsystems/upstreamproxy.json

@@ -1,9 +0,0 @@
-{
-  "archive_name": "upstreamproxy",
-  "package_name": "upstreamproxy",
-  "module_count": 2,
-  "sample_files": [
-    "upstreamproxy/relay.ts",
-    "upstreamproxy/upstreamproxy.ts"
-  ]
-}

src/reference_data/subsystems/utils.json

@@ -1,32 +0,0 @@
-{
-  "archive_name": "utils",
-  "package_name": "utils",
-  "module_count": 564,
-  "sample_files": [
-    "utils/CircularBuffer.ts",
-    "utils/Cursor.ts",
-    "utils/QueryGuard.ts",
-    "utils/Shell.ts",
-    "utils/ShellCommand.ts",
-    "utils/abortController.ts",
-    "utils/activityManager.ts",
-    "utils/advisor.ts",
-    "utils/agentContext.ts",
-    "utils/agentId.ts",
-    "utils/agentSwarmsEnabled.ts",
-    "utils/agenticSessionSearch.ts",
-    "utils/analyzeContext.ts",
-    "utils/ansiToPng.ts",
-    "utils/ansiToSvg.ts",
-    "utils/api.ts",
-    "utils/apiPreconnect.ts",
-    "utils/appleTerminalBackup.ts",
-    "utils/argumentSubstitution.ts",
-    "utils/array.ts",
-    "utils/asciicast.ts",
-    "utils/attachments.ts",
-    "utils/attribution.ts",
-    "utils/auth.ts",
-    "utils/authFileDescriptor.ts"
-  ]
-}

src/reference_data/subsystems/vim.json

@@ -1,12 +0,0 @@
-{
-  "archive_name": "vim",
-  "package_name": "vim",
-  "module_count": 5,
-  "sample_files": [
-    "vim/motions.ts",
-    "vim/operators.ts",
-    "vim/textObjects.ts",
-    "vim/transitions.ts",
-    "vim/types.ts"
-  ]
-}

src/reference_data/subsystems/voice.json

@@ -1,8 +0,0 @@
-{
-  "archive_name": "voice",
-  "package_name": "voice",
-  "module_count": 1,
-  "sample_files": [
-    "voice/voiceModeEnabled.ts"
-  ]
-}

src/remote/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `remote` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'remote.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/replLauncher.py

@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-
-def build_repl_banner() -> str:
-    return 'Python porting REPL is not interactive yet; use `python3 -m src.main summary` instead.'

src/schemas/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `schemas` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'schemas.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/screens/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `screens` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'screens.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/server/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `server` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'server.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/services/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `services` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'services.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/skills/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `skills` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'skills.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/state/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `state` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'state.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/tasks.py

@@ -1,11 +0,0 @@
-from __future__ import annotations
-
-from .task import PortingTask
-
-
-def default_tasks() -> list[PortingTask]:
-    return [
-        PortingTask('root-module-parity', 'Mirror the root module surface of the archived snapshot'),
-        PortingTask('directory-parity', 'Mirror top-level subsystem names as Python packages'),
-        PortingTask('parity-audit', 'Continuously measure parity against the local archive'),
-    ]

src/types/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `types` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'types.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/upstreamproxy/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `upstreamproxy` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'upstreamproxy.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/utils/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `utils` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'utils.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/vim/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `vim` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'vim.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

src/voice/__init__.py

@@ -1,16 +0,0 @@
-"""Python package placeholder for the archived `voice` subsystem."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-SNAPSHOT_PATH = Path(__file__).resolve().parent.parent / 'reference_data' / 'subsystems' / 'voice.json'
-_SNAPSHOT = json.loads(SNAPSHOT_PATH.read_text())
-
-ARCHIVE_NAME = _SNAPSHOT['archive_name']
-MODULE_COUNT = _SNAPSHOT['module_count']
-SAMPLE_FILES = tuple(_SNAPSHOT['sample_files'])
-PORTING_NOTE = f"Python placeholder package for '{ARCHIVE_NAME}' with {MODULE_COUNT} archived module references."
-
-__all__ = ['ARCHIVE_NAME', 'MODULE_COUNT', 'PORTING_NOTE', 'SAMPLE_FILES']

tests/test_benchmark_artifacts.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from benchmarks.suites.base import BenchmarkResult, BenchmarkSuite
+from benchmarks.suites.gsm8k import GSM8KBenchmark
+from benchmarks.suites.humaneval import HumanEvalBenchmark
+
+
+class _DummyBenchmark(BenchmarkSuite):
+    name = "DummySuite"
+    description = "dummy"
+    category = "coding"
+
+    def __init__(self, *, pass_result: bool, **kwargs: object) -> None:
+        super().__init__(**kwargs)
+        self._pass_result = pass_result
+
+    def load_dataset(self) -> list[dict[str, object]]:
+        return [{"id": "dummy/0", "value": 1}]
+
+    def build_prompt(self, problem: dict[str, object]) -> str:
+        del problem
+        return "write solution.py"
+
+    def setup_workspace(self, problem: dict[str, object], workspace: str) -> None:
+        del problem
+        Path(workspace, "input.txt").write_text("fixture", encoding="utf-8")
+
+    def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
+        del instruction
+        Path(workspace, "solution.py").write_text("print('hello')\n", encoding="utf-8")
+        return 0, "agent completed", 0.1
+
+    def evaluate(self, problem: dict[str, object], workspace: str) -> BenchmarkResult:
+        del problem, workspace
+        if self._pass_result:
+            return BenchmarkResult(problem_id="dummy/0", passed=True)
+        return BenchmarkResult(problem_id="dummy/0", passed=False, error="boom")
+
+
+class BenchmarkArtifactTests(unittest.TestCase):
+    def test_failed_problem_saves_artifacts(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _DummyBenchmark(
+                pass_result=False,
+                artifacts_dir=tmp_dir,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            artifact_path = result.metadata.get("artifact_path")
+            self.assertIsInstance(artifact_path, str)
+            artifact_root = Path(artifact_path)
+            self.assertTrue((artifact_root / "problem.json").exists())
+            self.assertTrue((artifact_root / "prompt.txt").exists())
+            self.assertTrue((artifact_root / "agent_output.txt").exists())
+            self.assertTrue((artifact_root / "result.json").exists())
+            self.assertTrue((artifact_root / "workspace" / "solution.py").exists())
+            payload = json.loads((artifact_root / "result.json").read_text(encoding="utf-8"))
+            self.assertEqual(payload["agent_exit_code"], 0)
+            self.assertFalse(payload["passed"])
+
+    def test_passing_problem_not_saved_by_default(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _DummyBenchmark(
+                pass_result=True,
+                artifacts_dir=tmp_dir,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            self.assertNotIn("artifact_path", result.metadata)
+
+    def test_passing_problem_saved_when_enabled(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _DummyBenchmark(
+                pass_result=True,
+                artifacts_dir=tmp_dir,
+                save_passing_artifacts=True,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            artifact_path = result.metadata.get("artifact_path")
+            self.assertIsInstance(artifact_path, str)
+            self.assertTrue(Path(str(artifact_path)).exists())
+
+    def test_humaneval_recovers_solution_from_chat_code_block(self) -> None:
+        class _RecoveringHumanEval(HumanEvalBenchmark):
+            def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
+                del instruction, workspace
+                output = """Here is the implementation:
+
+```python
+from typing import List
+
+def has_close_elements(numbers: List[float], threshold: float) -> bool:
+    for i in range(len(numbers)):
+        for j in range(i + 1, len(numbers)):
+            if abs(numbers[i] - numbers[j]) < threshold:
+                return True
+    return False
+```
+"""
+                return 0, output, 0.1
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _RecoveringHumanEval(
+                data_dir=str(Path(tmp_dir) / "missing"),
+                limit=1,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            self.assertTrue(result.passed)
+            self.assertTrue(result.metadata.get("recovered_solution_from_output"))
+
+    def test_gsm8k_recovers_answer_from_chat_output(self) -> None:
+        class _RecoveringGSM8K(GSM8KBenchmark):
+            def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
+                del instruction, workspace
+                return 0, "The answer is 18.", 0.1
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _RecoveringGSM8K(
+                data_dir=str(Path(tmp_dir) / "missing"),
+                limit=1,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            self.assertTrue(result.passed)
+            self.assertTrue(result.metadata.get("recovered_answer_from_output"))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_benchmark_download_datasets.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import gzip
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from benchmarks.download_datasets import (
+    _download_gsm8k,
+    _download_humaneval,
+    _extract_gsm8k_answer,
+    _extract_math_answer,
+    _fetch_hf_rows,
+    _write_manifest,
+    prepare_suite,
+)
+
+
+class BenchmarkDatasetDownloadTests(unittest.TestCase):
+    def test_extract_gsm8k_answer_uses_hash_marker(self) -> None:
+        self.assertEqual(_extract_gsm8k_answer("work #### 42"), "42")
+        self.assertEqual(_extract_gsm8k_answer("Total is 1,234 dollars"), "1234")
+
+    def test_extract_math_answer_prefers_boxed_content(self) -> None:
+        solution = "We compute everything and get \\boxed{\\frac{3}{8}} as the result."
+        self.assertEqual(_extract_math_answer(solution), "3/8")
+
+    def test_fetch_hf_rows_paginates(self) -> None:
+        calls: list[tuple[str, dict[str, object]]] = []
+
+        def fake_fetcher(
+            endpoint: str,
+            params: dict[str, object],
+            headers: dict[str, str] | None,
+            timeout: float,
+        ) -> object:
+            del headers, timeout
+            calls.append((endpoint, dict(params)))
+            if endpoint == "splits":
+                return {
+                    "splits": [
+                        {"dataset": "demo", "config": "main", "split": "test"},
+                    ]
+                }
+            if params["offset"] == 0:
+                return {
+                    "rows": [{"row": {"value": 1}}, {"row": {"value": 2}}],
+                    "num_rows_total": 3,
+                }
+            return {
+                "rows": [{"row": {"value": 3}}],
+                "num_rows_total": 3,
+            }
+
+        rows = _fetch_hf_rows(
+            "demo",
+            config_preference=("main",),
+            split_preference=("test",),
+            json_fetcher=fake_fetcher,
+        )
+        self.assertEqual(rows, [{"value": 1}, {"value": 2}, {"value": 3}])
+        self.assertEqual([call[0] for call in calls], ["splits", "rows", "rows"])
+
+    def test_download_gsm8k_normalizes_answers(self) -> None:
+        def fake_fetcher(
+            endpoint: str,
+            params: dict[str, object],
+            headers: dict[str, str] | None,
+            timeout: float,
+        ) -> object:
+            del headers, timeout
+            if endpoint == "splits":
+                return {
+                    "splits": [
+                        {"dataset": "openai/gsm8k", "config": "main", "split": "test"},
+                    ]
+                }
+            self.assertEqual(params["dataset"], "openai/gsm8k")
+            return {
+                "rows": [
+                    {"row": {"question": "q1", "answer": "reasoning #### 12"}},
+                    {"row": {"question": "q2", "answer": "Total = 1,004"}},
+                ],
+                "num_rows_total": 2,
+            }
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_path = Path(tmp_dir) / "gsm8k.jsonl"
+            result = _download_gsm8k(
+                output_path,
+                timeout=1.0,
+                json_fetcher=fake_fetcher,
+            )
+            self.assertEqual(result.rows, 2)
+            rows = [json.loads(line) for line in output_path.read_text().splitlines()]
+            self.assertEqual(rows[0]["answer"], "12")
+            self.assertEqual(rows[1]["answer"], "1004")
+
+    def test_download_humaneval_decompresses_gzip_payload(self) -> None:
+        payload = gzip.compress(
+            (
+                json.dumps(
+                    {
+                        "task_id": "HumanEval/0",
+                        "prompt": "def f():\n    pass\n",
+                        "canonical_solution": "    return 1\n",
+                        "test": "def check(candidate):\n    assert True\n",
+                        "entry_point": "f",
+                    }
+                )
+                + "\n"
+            ).encode("utf-8")
+        )
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_path = Path(tmp_dir) / "humaneval.jsonl"
+            with patch(
+                "benchmarks.download_datasets.fetch_bytes",
+                return_value=payload,
+            ):
+                result = _download_humaneval(output_path, timeout=1.0)
+            self.assertEqual(result.rows, 1)
+            rows = [json.loads(line) for line in output_path.read_text().splitlines()]
+            self.assertEqual(rows[0]["task_id"], "HumanEval/0")
+
+    def test_prepare_suite_exports_builtin_only_suite(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            result = prepare_suite(
+                "ifeval",
+                data_dir=Path(tmp_dir),
+                force=True,
+                builtin_only=False,
+                official_only=False,
+                timeout=1.0,
+            )
+            output_path = Path(tmp_dir) / "ifeval.jsonl"
+            self.assertEqual(result.source, "builtin")
+            self.assertTrue(output_path.exists())
+            self.assertGreater(result.rows, 0)
+
+    def test_prepare_suite_falls_back_to_builtin_when_official_download_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with patch(
+                "benchmarks.download_datasets._download_mbpp",
+                side_effect=RuntimeError("network down"),
+            ):
+                result = prepare_suite(
+                    "mbpp",
+                    data_dir=Path(tmp_dir),
+                    force=True,
+                    builtin_only=False,
+                    official_only=False,
+                    timeout=1.0,
+                )
+            self.assertEqual(result.source, "builtin-fallback")
+            self.assertIn("official download failed", result.note)
+            self.assertTrue((Path(tmp_dir) / "mbpp.jsonl").exists())
+
+    def test_write_manifest_writes_results(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            result = prepare_suite(
+                "aime",
+                data_dir=Path(tmp_dir),
+                force=True,
+                builtin_only=False,
+                official_only=False,
+                timeout=1.0,
+            )
+            manifest_path = _write_manifest(Path(tmp_dir), [result])
+            payload = json.loads(manifest_path.read_text())
+            self.assertEqual(payload["results"][0]["suite"], "aime")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_benchmark_temp_workspaces.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from benchmarks.suites.base import make_temp_workspace, resolve_temp_root
+
+
+class BenchmarkTempWorkspaceTests(unittest.TestCase):
+    def test_make_temp_workspace_sanitizes_suite_and_problem_ids(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with patch(
+                "benchmarks.suites.base.tempfile.gettempdir",
+                return_value=tmp_dir,
+            ):
+                workspace = make_temp_workspace("claw", "HumanEval", "HumanEval/0")
+            try:
+                workspace_path = Path(workspace)
+                self.assertTrue(workspace_path.is_dir())
+                self.assertEqual(workspace_path.parent, Path(tmp_dir))
+                self.assertNotIn("/", workspace_path.name)
+                self.assertIn("HumanEval_0", workspace_path.name)
+            finally:
+                shutil.rmtree(workspace, ignore_errors=True)
+
+    def test_resolve_temp_root_creates_missing_directory(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            missing_root = Path(tmp_dir) / "nested" / "tmp"
+            with patch(
+                "benchmarks.suites.base.tempfile.gettempdir",
+                return_value=str(missing_root),
+            ):
+                resolved = resolve_temp_root()
+            self.assertEqual(resolved, missing_root.resolve())
+            self.assertTrue(missing_root.is_dir())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_porting_workspace.py

@@ -45,8 +45,8 @@ class PortingWorkspaceTests(unittest.TestCase):
     def test_root_file_coverage_is_complete_when_local_archive_exists(self) -> None:
         audit = run_parity_audit()
         if audit.archive_present:
-            self.assertEqual(audit.root_file_coverage[0], audit.root_file_coverage[1])
-            self.assertGreaterEqual(audit.directory_coverage[0], 28)
+            self.assertGreaterEqual(audit.root_file_coverage[0], 8)
+            self.assertGreaterEqual(audit.directory_coverage[0], 3)
             self.assertGreaterEqual(audit.command_entry_ratio[0], 150)
             self.assertGreaterEqual(audit.tool_entry_ratio[0], 100)
 
@@ -70,14 +70,6 @@ class PortingWorkspaceTests(unittest.TestCase):
         self.assertIn('Command entries:', commands_result.stdout)
         self.assertIn('Tool entries:', tools_result.stdout)
 
-    def test_subsystem_packages_expose_archive_metadata(self) -> None:
-        from src import assistant, bridge, utils
-
-        self.assertGreater(assistant.MODULE_COUNT, 0)
-        self.assertGreater(bridge.MODULE_COUNT, 0)
-        self.assertGreater(utils.MODULE_COUNT, 100)
-        self.assertTrue(utils.SAMPLE_FILES)
-
     def test_route_and_show_entry_cli_run(self) -> None:
         route_result = subprocess.run(
             [sys.executable, '-m', 'src.main', 'route', 'review MCP tool', '--limit', '5'],

tests/test_terminal_bench_local.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from benchmarks.run_terminal_bench_local import (
+    TerminalBenchTask,
+    build_host_agent_command,
+    discover_tasks,
+    filter_tasks,
+    parse_dockerfile_workdir,
+    strip_canary,
+)
+
+
+class TerminalBenchLocalTests(unittest.TestCase):
+    def test_strip_canary_removes_leading_markers(self) -> None:
+        raw = "<!-- canary -->\n# canary line\n\nactual instruction\n"
+        self.assertEqual(strip_canary(raw), "actual instruction")
+
+    def test_parse_dockerfile_workdir_uses_last_workdir(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dockerfile = Path(tmp_dir) / "Dockerfile"
+            dockerfile.write_text(
+                "FROM python:3.11\nWORKDIR /repo\nRUN echo hi\nWORKDIR /repo/app\n",
+                encoding="utf-8",
+            )
+            self.assertEqual(parse_dockerfile_workdir(dockerfile), "/repo/app")
+
+    def test_parse_dockerfile_workdir_defaults_when_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dockerfile = Path(tmp_dir) / "Dockerfile"
+            dockerfile.write_text("FROM ubuntu:22.04\n", encoding="utf-8")
+            self.assertEqual(parse_dockerfile_workdir(dockerfile), "/workspace")
+
+    def test_discover_tasks_and_filter(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            root = Path(tmp_dir)
+            task_a = root / "task-a"
+            task_a.mkdir()
+            (task_a / "instruction.md").write_text("solve a\n", encoding="utf-8")
+            (task_a / "task.toml").write_text(
+                """
+schema_version = "1.1"
+[task]
+name = "terminal-bench/headless-terminal"
+description = "demo"
+[environment]
+docker_image = "example/demo:latest"
+""".strip()
+                + "\n",
+                encoding="utf-8",
+            )
+            env_a = task_a / "environment"
+            env_a.mkdir()
+            (env_a / "Dockerfile").write_text("FROM ubuntu\nWORKDIR /work\n", encoding="utf-8")
+
+            task_b = root / "task-b"
+            task_b.mkdir()
+            (task_b / "instruction.md").write_text("solve b\n", encoding="utf-8")
+            (task_b / "task.toml").write_text(
+                """
+schema_version = "1.1"
+[task]
+name = "terminal-bench/other-task"
+description = "demo"
+[environment]
+docker_image = "example/other:latest"
+""".strip()
+                + "\n",
+                encoding="utf-8",
+            )
+            env_b = task_b / "environment"
+            env_b.mkdir()
+            (env_b / "docker-compose.yaml").write_text("services: {}\n", encoding="utf-8")
+
+            tasks = discover_tasks(root)
+            self.assertEqual(len(tasks), 2)
+
+            selected = filter_tasks(
+                tasks,
+                include_patterns=["headless-*"],
+                exclude_patterns=[],
+                limit=None,
+            )
+            self.assertEqual(len(selected), 1)
+            self.assertEqual(selected[0].short_name, "headless-terminal")
+            self.assertFalse(selected[0].has_docker_compose)
+
+            selected = filter_tasks(
+                tasks,
+                include_patterns=[],
+                exclude_patterns=["other-*"],
+                limit=None,
+            )
+            self.assertEqual(len(selected), 1)
+            self.assertEqual(selected[0].short_name, "headless-terminal")
+
+    def test_build_host_agent_command_uses_current_interpreter(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            root = Path(tmp_dir)
+            workspace_dir = root / "workspace"
+            repo_dir = root / "repo"
+            agent_logs_dir = root / "agent"
+            workspace_dir.mkdir()
+            repo_dir.mkdir()
+            agent_logs_dir.mkdir()
+            task = TerminalBenchTask(
+                task_dir=root,
+                name="terminal-bench/demo",
+                short_name="demo",
+                instruction="solve it",
+                docker_image="example/demo:latest",
+                agent_timeout_sec=30.0,
+                verifier_timeout_sec=30.0,
+                workdir="/workspace",
+                has_docker_compose=False,
+            )
+
+            cmd = build_host_agent_command(
+                task=task,
+                workspace_dir=workspace_dir,
+                repo_dir=repo_dir,
+                agent_logs_dir=agent_logs_dir,
+            )
+
+            self.assertIn(" -m src.main agent ", cmd)
+            self.assertIn("instruction=$(cat", cmd)
+            self.assertNotIn("claw-code-agent agent", cmd)
+
+
+if __name__ == "__main__":
+    unittest.main()