From a54c90b18f75fcc46b19017ff55d30aff1c74aa0 Mon Sep 17 00:00:00 2001 From: Abdelrahman Abdallah Date: Mon, 6 Apr 2026 03:42:44 +0200 Subject: [PATCH] update the codebase and clean up it --- .gitignore | 9 +- benchmarks/download_datasets.py | 391 +++++++++++++ benchmarks/run.py | 3 +- benchmarks/run_suite.py | 6 + benchmarks/run_terminal_bench_local.py | 512 ++++++++++++++++++ benchmarks/suites/aime.py | 16 + benchmarks/suites/base.py | 203 ++++--- benchmarks/suites/gsm8k.py | 16 + benchmarks/suites/humaneval.py | 38 +- benchmarks/suites/ifeval.py | 16 + benchmarks/suites/math_bench.py | 16 + benchmarks/suites/mbpp.py | 36 +- harbor_adapter.py | 95 ++++ install_dockor.sh | 36 ++ pyproject.toml | 1 - src/QueryEngine.py | 19 - src/Tool.py | 15 - src/assistant/__init__.py | 16 - src/bootstrap/__init__.py | 16 - src/bridge/__init__.py | 16 - src/buddy/__init__.py | 16 - src/cli/__init__.py | 16 - src/components/__init__.py | 16 - src/constants/__init__.py | 16 - src/coordinator/__init__.py | 16 - src/costHook.py | 8 - src/dialogLaunchers.py | 15 - src/direct_modes.py | 21 - src/entrypoints/__init__.py | 16 - src/hooks/__init__.py | 16 - src/ink.py | 6 - src/interactiveHelpers.py | 5 - src/keybindings/__init__.py | 16 - src/memdir/__init__.py | 16 - src/migrations/__init__.py | 16 - src/moreright/__init__.py | 16 - src/native_ts/__init__.py | 16 - src/outputStyles/__init__.py | 16 - src/plugins/__init__.py | 16 - src/projectOnboardingState.py | 10 - src/query.py | 13 - src/reference_data/subsystems/assistant.json | 8 - src/reference_data/subsystems/bootstrap.json | 8 - src/reference_data/subsystems/bridge.json | 32 -- src/reference_data/subsystems/buddy.json | 13 - src/reference_data/subsystems/cli.json | 26 - src/reference_data/subsystems/components.json | 32 -- src/reference_data/subsystems/constants.json | 28 - .../subsystems/coordinator.json | 8 - .../subsystems/entrypoints.json | 15 - src/reference_data/subsystems/hooks.json | 32 -- .../subsystems/keybindings.json | 21 - src/reference_data/subsystems/memdir.json | 15 - src/reference_data/subsystems/migrations.json | 18 - src/reference_data/subsystems/moreright.json | 8 - src/reference_data/subsystems/native_ts.json | 11 - .../subsystems/outputStyles.json | 8 - src/reference_data/subsystems/plugins.json | 9 - src/reference_data/subsystems/remote.json | 11 - src/reference_data/subsystems/schemas.json | 8 - src/reference_data/subsystems/screens.json | 10 - src/reference_data/subsystems/server.json | 10 - src/reference_data/subsystems/services.json | 32 -- src/reference_data/subsystems/skills.json | 27 - src/reference_data/subsystems/state.json | 13 - src/reference_data/subsystems/types.json | 18 - .../subsystems/upstreamproxy.json | 9 - src/reference_data/subsystems/utils.json | 32 -- src/reference_data/subsystems/vim.json | 12 - src/reference_data/subsystems/voice.json | 8 - src/remote/__init__.py | 16 - src/replLauncher.py | 5 - src/schemas/__init__.py | 16 - src/screens/__init__.py | 16 - src/server/__init__.py | 16 - src/services/__init__.py | 16 - src/skills/__init__.py | 16 - src/state/__init__.py | 16 - src/tasks.py | 11 - src/types/__init__.py | 16 - src/upstreamproxy/__init__.py | 16 - src/utils/__init__.py | 16 - src/vim/__init__.py | 16 - src/voice/__init__.py | 16 - tests/test_benchmark_artifacts.py | 136 +++++ tests/test_benchmark_download_datasets.py | 177 ++++++ tests/test_benchmark_temp_workspaces.py | 42 ++ tests/test_porting_workspace.py | 26 +- tests/test_terminal_bench_local.py | 134 +++++ 89 files changed, 1802 insertions(+), 1181 deletions(-) create mode 100644 benchmarks/download_datasets.py create mode 100644 benchmarks/run_terminal_bench_local.py create mode 100644 harbor_adapter.py create mode 100644 install_dockor.sh delete mode 100644 src/QueryEngine.py delete mode 100644 src/Tool.py delete mode 100644 src/assistant/__init__.py delete mode 100644 src/bootstrap/__init__.py delete mode 100644 src/bridge/__init__.py delete mode 100644 src/buddy/__init__.py delete mode 100644 src/cli/__init__.py delete mode 100644 src/components/__init__.py delete mode 100644 src/constants/__init__.py delete mode 100644 src/coordinator/__init__.py delete mode 100644 src/costHook.py delete mode 100644 src/dialogLaunchers.py delete mode 100644 src/direct_modes.py delete mode 100644 src/entrypoints/__init__.py delete mode 100644 src/hooks/__init__.py delete mode 100644 src/ink.py delete mode 100644 src/interactiveHelpers.py delete mode 100644 src/keybindings/__init__.py delete mode 100644 src/memdir/__init__.py delete mode 100644 src/migrations/__init__.py delete mode 100644 src/moreright/__init__.py delete mode 100644 src/native_ts/__init__.py delete mode 100644 src/outputStyles/__init__.py delete mode 100644 src/plugins/__init__.py delete mode 100644 src/projectOnboardingState.py delete mode 100644 src/query.py delete mode 100644 src/reference_data/subsystems/assistant.json delete mode 100644 src/reference_data/subsystems/bootstrap.json delete mode 100644 src/reference_data/subsystems/bridge.json delete mode 100644 src/reference_data/subsystems/buddy.json delete mode 100644 src/reference_data/subsystems/cli.json delete mode 100644 src/reference_data/subsystems/components.json delete mode 100644 src/reference_data/subsystems/constants.json delete mode 100644 src/reference_data/subsystems/coordinator.json delete mode 100644 src/reference_data/subsystems/entrypoints.json delete mode 100644 src/reference_data/subsystems/hooks.json delete mode 100644 src/reference_data/subsystems/keybindings.json delete mode 100644 src/reference_data/subsystems/memdir.json delete mode 100644 src/reference_data/subsystems/migrations.json delete mode 100644 src/reference_data/subsystems/moreright.json delete mode 100644 src/reference_data/subsystems/native_ts.json delete mode 100644 src/reference_data/subsystems/outputStyles.json delete mode 100644 src/reference_data/subsystems/plugins.json delete mode 100644 src/reference_data/subsystems/remote.json delete mode 100644 src/reference_data/subsystems/schemas.json delete mode 100644 src/reference_data/subsystems/screens.json delete mode 100644 src/reference_data/subsystems/server.json delete mode 100644 src/reference_data/subsystems/services.json delete mode 100644 src/reference_data/subsystems/skills.json delete mode 100644 src/reference_data/subsystems/state.json delete mode 100644 src/reference_data/subsystems/types.json delete mode 100644 src/reference_data/subsystems/upstreamproxy.json delete mode 100644 src/reference_data/subsystems/utils.json delete mode 100644 src/reference_data/subsystems/vim.json delete mode 100644 src/reference_data/subsystems/voice.json delete mode 100644 src/remote/__init__.py delete mode 100644 src/replLauncher.py delete mode 100644 src/schemas/__init__.py delete mode 100644 src/screens/__init__.py delete mode 100644 src/server/__init__.py delete mode 100644 src/services/__init__.py delete mode 100644 src/skills/__init__.py delete mode 100644 src/state/__init__.py delete mode 100644 src/tasks.py delete mode 100644 src/types/__init__.py delete mode 100644 src/upstreamproxy/__init__.py delete mode 100644 src/utils/__init__.py delete mode 100644 src/vim/__init__.py delete mode 100644 src/voice/__init__.py create mode 100644 tests/test_benchmark_artifacts.py create mode 100644 tests/test_benchmark_download_datasets.py create mode 100644 tests/test_benchmark_temp_workspaces.py create mode 100644 tests/test_terminal_bench_local.py diff --git a/.gitignore b/.gitignore index 5bdc17d..1c4a2cf 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,13 @@ archive/ .env .env.* +# Local benchmark outputs +benchmark_artifacts/ +jobs/ +output_terminal/ +tb2*.json +humaneval_results.json + test_cases -e-commerce \ No newline at end of file +e-commerce diff --git a/benchmarks/download_datasets.py b/benchmarks/download_datasets.py new file mode 100644 index 0000000..16075f2 --- /dev/null +++ b/benchmarks/download_datasets.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +""" +Download or export benchmark datasets into benchmarks/data. +""" + +from __future__ import annotations + +import argparse +import gzip +import json +import re +import urllib.parse +import urllib.request +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable + +from benchmarks.suites.aider import _BUILTIN_PROBLEMS as _AIDER_BUILTINS +from benchmarks.suites.aime import _BUILTIN_PROBLEMS as _AIME_BUILTINS +from benchmarks.suites.bfcl import _BUILTIN_PROBLEMS as _BFCL_BUILTINS +from benchmarks.suites.gsm8k import _BUILTIN_PROBLEMS as _GSM8K_BUILTINS +from benchmarks.suites.humaneval import _BUILTIN_PROBLEMS as _HUMANEVAL_BUILTINS +from benchmarks.suites.ifeval import _BUILTIN_PROBLEMS as _IFEVAL_BUILTINS +from benchmarks.suites.livecodebench import _BUILTIN_PROBLEMS as _LIVECODEBENCH_BUILTINS +from benchmarks.suites.math_bench import _BUILTIN_PROBLEMS as _MATH_BUILTINS +from benchmarks.suites.mbpp import _BUILTIN_PROBLEMS as _MBPP_BUILTINS +from benchmarks.suites.swe_bench import _BUILTIN_PROBLEMS as _SWE_BUILTINS + + +HF_DATASET_VIEWER_BASE = "https://datasets-server.huggingface.co" +HUMANEVAL_GZ_URL = "https://raw.githubusercontent.com/openai/human-eval/master/data/HumanEval.jsonl.gz" +DEFAULT_DATA_DIR = Path(__file__).resolve().parent / "data" + +JsonFetcher = Callable[[str, dict[str, object], dict[str, str] | None, float], object] + + +@dataclass +class DownloadResult: + suite: str + rows: int + path: str + source: str + note: str = "" + + +def fetch_bytes(url: str, timeout: float, headers: dict[str, str] | None = None) -> bytes: + request = urllib.request.Request(url, headers=headers or {}) + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + + +def fetch_json( + endpoint: str, + params: dict[str, object], + headers: dict[str, str] | None, + timeout: float, +) -> object: + query = urllib.parse.urlencode(params, doseq=True) + url = f"{HF_DATASET_VIEWER_BASE}/{endpoint}" + if query: + url = f"{url}?{query}" + raw = fetch_bytes(url, timeout=timeout, headers=headers) + return json.loads(raw.decode("utf-8")) + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> int: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=True) + "\n") + return len(rows) + + +def _extract_gsm8k_answer(text: str) -> str: + if "####" in text: + text = text.split("####", 1)[1] + numbers = re.findall(r"-?\d[\d,]*\.?\d*", text.replace("$", "")) + if numbers: + return numbers[-1].replace(",", "") + return text.strip().replace(",", "") + + +def _extract_math_answer(solution: str) -> str: + boxed_fraction = re.search(r"\\boxed\{\\frac\{([^}]+)\}\{([^}]+)\}\}", solution, flags=re.DOTALL) + if boxed_fraction: + return f"{boxed_fraction.group(1).strip()}/{boxed_fraction.group(2).strip()}" + boxed = re.search(r"\\boxed\{([^{}]+)\}", solution, flags=re.DOTALL) + value = boxed.group(1) if boxed else solution + value = value.strip() + value = value.replace("\\frac{", "").replace("}{", "/").replace("}", "") + value = value.replace("$", "").replace(",", "").strip() + fraction = re.search(r"-?\d+\s*/\s*-?\d+", value) + if fraction: + return fraction.group(0).replace(" ", "") + numbers = re.findall(r"-?\d+(?:/\d+)?(?:\.\d+)?", value) + return numbers[-1] if numbers else value + + +def _fetch_hf_rows( + dataset: str, + *, + config_preference: tuple[str, ...] = (), + split_preference: tuple[str, ...] = ("test", "validation", "train"), + json_fetcher: JsonFetcher = fetch_json, + timeout: float = 60.0, + headers: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + splits_payload = json_fetcher("splits", {"dataset": dataset}, headers, timeout) + splits = list((splits_payload or {}).get("splits", [])) # type: ignore[assignment] + if not splits: + return [] + + chosen: dict[str, Any] | None = None + for config_name in config_preference: + for split_name in split_preference: + chosen = next( + ( + item for item in splits + if item.get("config") == config_name and item.get("split") == split_name + ), + None, + ) + if chosen is not None: + break + if chosen is not None: + break + if chosen is None: + for split_name in split_preference: + chosen = next((item for item in splits if item.get("split") == split_name), None) + if chosen is not None: + break + if chosen is None: + chosen = splits[0] + + rows: list[dict[str, Any]] = [] + offset = 0 + length = 100 + while True: + payload = json_fetcher( + "rows", + { + "dataset": chosen["dataset"], + "config": chosen["config"], + "split": chosen["split"], + "offset": offset, + "length": length, + }, + headers, + timeout, + ) + batch = [item.get("row", {}) for item in (payload or {}).get("rows", [])] # type: ignore[union-attr] + rows.extend(batch) + total = int((payload or {}).get("num_rows_total", len(rows))) # type: ignore[union-attr] + offset += len(batch) + if not batch or offset >= total: + break + return rows + + +def _download_humaneval(output_path: Path, *, timeout: float) -> DownloadResult: + raw = fetch_bytes(HUMANEVAL_GZ_URL, timeout=timeout) + if raw[:2] == b"\x1f\x8b": + raw = gzip.decompress(raw) + lines = [json.loads(line) for line in raw.decode("utf-8").splitlines() if line.strip()] + rows = [ + { + "task_id": item["task_id"], + "prompt": item["prompt"], + "canonical_solution": item.get("canonical_solution", ""), + "test": item["test"], + "entry_point": item["entry_point"], + } + for item in lines + ] + count = _write_jsonl(output_path, rows) + return DownloadResult("humaneval", count, str(output_path), "official") + + +def _download_gsm8k( + output_path: Path, + *, + timeout: float, + json_fetcher: JsonFetcher = fetch_json, +) -> DownloadResult: + rows = _fetch_hf_rows( + "openai/gsm8k", + config_preference=("main",), + split_preference=("test",), + json_fetcher=json_fetcher, + timeout=timeout, + ) + normalized = [ + { + "id": f"gsm8k-{index + 1:04d}", + "question": row["question"], + "answer": _extract_gsm8k_answer(str(row["answer"])), + } + for index, row in enumerate(rows) + ] + count = _write_jsonl(output_path, normalized) + return DownloadResult("gsm8k", count, str(output_path), "official") + + +def _download_mbpp( + output_path: Path, + *, + timeout: float, + json_fetcher: JsonFetcher = fetch_json, +) -> DownloadResult: + rows = _fetch_hf_rows( + "google-research-datasets/mbpp", + config_preference=("sanitized", "full"), + split_preference=("test", "validation"), + json_fetcher=json_fetcher, + timeout=timeout, + ) + normalized = [ + { + "task_id": row.get("task_id", index + 1), + "text": row.get("text") or row.get("prompt") or "", + "code": row.get("code", ""), + "test_list": row.get("test_list") or row.get("test_setup_code", []), + } + for index, row in enumerate(rows) + ] + count = _write_jsonl(output_path, normalized) + return DownloadResult("mbpp", count, str(output_path), "official") + + +def _download_math( + output_path: Path, + *, + timeout: float, + json_fetcher: JsonFetcher = fetch_json, +) -> DownloadResult: + rows = _fetch_hf_rows( + "hendrycks/competition_math", + config_preference=("default",), + split_preference=("test", "train"), + json_fetcher=json_fetcher, + timeout=timeout, + ) + normalized = [ + { + "id": row.get("problem_id", f"math-{index + 1:04d}"), + "problem": row.get("problem", ""), + "answer": _extract_math_answer(str(row.get("solution", row.get("answer", "")))), + "subject": row.get("type", row.get("subject", "unknown")), + "level": row.get("level", 0), + } + for index, row in enumerate(rows) + ] + count = _write_jsonl(output_path, normalized) + return DownloadResult("math", count, str(output_path), "official") + + +def _export_builtin(output_path: Path, suite: str, rows: list[dict[str, Any]], *, source: str = "builtin", note: str = "") -> DownloadResult: + count = _write_jsonl(output_path, rows) + return DownloadResult(suite, count, str(output_path), source, note) + + +def _builtin_rows(suite: str) -> list[dict[str, Any]]: + mapping: dict[str, list[dict[str, Any]]] = { + "humaneval": list(_HUMANEVAL_BUILTINS), + "mbpp": list(_MBPP_BUILTINS), + "gsm8k": list(_GSM8K_BUILTINS), + "math": list(_MATH_BUILTINS), + "swe-bench": list(_SWE_BUILTINS), + "aider": list(_AIDER_BUILTINS), + "livecodebench": list(_LIVECODEBENCH_BUILTINS), + "aime": list(_AIME_BUILTINS), + "ifeval": list(_IFEVAL_BUILTINS), + "bfcl": list(_BFCL_BUILTINS), + } + return list(mapping[suite]) + + +def prepare_suite( + suite: str, + *, + data_dir: Path, + force: bool, + builtin_only: bool, + official_only: bool, + timeout: float, +) -> DownloadResult: + output_path = data_dir / f"{suite}.jsonl" + if output_path.exists() and not force: + lines = [line for line in output_path.read_text(encoding="utf-8").splitlines() if line.strip()] + return DownloadResult(suite, len(lines), str(output_path), "existing") + + data_dir.mkdir(parents=True, exist_ok=True) + + builtin_only_suites = {"swe-bench", "aider", "livecodebench", "aime", "ifeval", "bfcl"} + official_downloaders = { + "humaneval": _download_humaneval, + "gsm8k": _download_gsm8k, + "mbpp": _download_mbpp, + "math": _download_math, + } + + if suite in builtin_only_suites or builtin_only: + return _export_builtin(output_path, suite, _builtin_rows(suite)) + + downloader = official_downloaders.get(suite) + if downloader is None: + return _export_builtin(output_path, suite, _builtin_rows(suite)) + + try: + return downloader(output_path, timeout=timeout) + except Exception as exc: + if official_only: + raise + note = f"official download failed: {exc}" + return _export_builtin( + output_path, + suite, + _builtin_rows(suite), + source="builtin-fallback", + note=note, + ) + + +def _write_manifest(data_dir: Path, results: list[DownloadResult]) -> Path: + manifest_path = data_dir / "manifest.json" + payload = { + "generated_at": __import__("time").strftime("%Y-%m-%dT%H:%M:%S"), + "results": [asdict(item) for item in results], + } + manifest_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return manifest_path + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Download benchmark datasets for claw-code-agent.") + parser.add_argument("--suite", action="append", default=[], help="Suite to prepare. Can be repeated.") + parser.add_argument("--all", action="store_true", help="Prepare all known suites.") + parser.add_argument("--list", action="store_true", help="List known suites.") + parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR, help="Output data directory.") + parser.add_argument("--force", action="store_true", help="Overwrite existing files.") + parser.add_argument("--builtin-only", action="store_true", help="Skip official downloads and export builtins only.") + parser.add_argument("--official-only", action="store_true", help="Do not fall back to builtins.") + parser.add_argument("--timeout", type=float, default=60.0, help="Network timeout in seconds.") + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + known = [ + "humaneval", + "mbpp", + "gsm8k", + "math", + "swe-bench", + "aider", + "livecodebench", + "aime", + "ifeval", + "bfcl", + ] + + if args.list: + for name in known: + print(name) + return + + suites = list(args.suite) + if args.all: + suites = known + if not suites: + parser.error("specify --suite or --all") + + results = [ + prepare_suite( + suite, + data_dir=args.data_dir, + force=args.force, + builtin_only=args.builtin_only, + official_only=args.official_only, + timeout=args.timeout, + ) + for suite in suites + ] + manifest = _write_manifest(args.data_dir, results) + print(f"Wrote {len(results)} suite files to {args.data_dir}") + print(f"Manifest: {manifest}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/run.py b/benchmarks/run.py index dff6079..c478601 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -39,6 +39,7 @@ from dataclasses import dataclass from pathlib import Path from benchmarks.tasks.definitions import TASKS, BenchmarkTask, get_task +from benchmarks.suites.base import make_temp_workspace # --------------------------------------------------------------------------- @@ -90,7 +91,7 @@ def run_task( """Run a single benchmark task end-to-end.""" # Create isolated temp workspace - workspace = tempfile.mkdtemp(prefix=f"claw_bench_{task.id}_") + workspace = make_temp_workspace("claw_bench", task.category, task.id) if verbose: print(f" workspace: {workspace}") diff --git a/benchmarks/run_suite.py b/benchmarks/run_suite.py index b8178ee..c3705cf 100644 --- a/benchmarks/run_suite.py +++ b/benchmarks/run_suite.py @@ -193,6 +193,10 @@ def main() -> None: help="Save results to JSON file") parser.add_argument("--data-dir", help="Directory containing dataset files (JSONL)") + parser.add_argument("--artifacts-dir", + help="Directory where per-problem artifacts will be saved") + parser.add_argument("--save-passing-artifacts", action="store_true", + help="Also save artifacts for passing problems") args = parser.parse_args() if args.list: @@ -236,6 +240,8 @@ def main() -> None: limit=args.limit, agent_timeout=args.timeout, verbose=args.verbose, + artifacts_dir=args.artifacts_dir, + save_passing_artifacts=args.save_passing_artifacts, ) report = suite.run_all() reports.append(report) diff --git a/benchmarks/run_terminal_bench_local.py b/benchmarks/run_terminal_bench_local.py new file mode 100644 index 0000000..fa3dd89 --- /dev/null +++ b/benchmarks/run_terminal_bench_local.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +""" +Run Terminal-Bench tasks locally with Apptainer and claw-code-agent. + +This runner is designed for cluster setups where Docker is unavailable but +Apptainer is available and the model server runs on the same node. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +import tomllib +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + + +DEFAULT_TASKS_DIR = Path.home() / ".cache/harbor/tasks/packages/terminal-bench" +DEFAULT_JOBS_DIR = Path("jobs/terminal_bench_local") + + +@dataclass +class TerminalBenchTask: + task_dir: Path + name: str + short_name: str + instruction: str + docker_image: str | None + agent_timeout_sec: float | None + verifier_timeout_sec: float + workdir: str + has_docker_compose: bool + raw_config: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class LocalTrialResult: + task_name: str + short_name: str + passed: bool + reward: float | None + duration_sec: float + status: str + image: str | None + workdir: str + trial_dir: str + error: str = "" + agent_return_code: int | None = None + verifier_return_code: int | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +def _load_toml(path: Path) -> dict[str, Any]: + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def strip_canary(text: str) -> str: + lines = text.splitlines() + index = 0 + while index < len(lines): + stripped = lines[index].strip() + if stripped.startswith("\n# canary line\n\nactual instruction\n" + self.assertEqual(strip_canary(raw), "actual instruction") + + def test_parse_dockerfile_workdir_uses_last_workdir(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + dockerfile = Path(tmp_dir) / "Dockerfile" + dockerfile.write_text( + "FROM python:3.11\nWORKDIR /repo\nRUN echo hi\nWORKDIR /repo/app\n", + encoding="utf-8", + ) + self.assertEqual(parse_dockerfile_workdir(dockerfile), "/repo/app") + + def test_parse_dockerfile_workdir_defaults_when_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + dockerfile = Path(tmp_dir) / "Dockerfile" + dockerfile.write_text("FROM ubuntu:22.04\n", encoding="utf-8") + self.assertEqual(parse_dockerfile_workdir(dockerfile), "/workspace") + + def test_discover_tasks_and_filter(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + task_a = root / "task-a" + task_a.mkdir() + (task_a / "instruction.md").write_text("solve a\n", encoding="utf-8") + (task_a / "task.toml").write_text( + """ +schema_version = "1.1" +[task] +name = "terminal-bench/headless-terminal" +description = "demo" +[environment] +docker_image = "example/demo:latest" +""".strip() + + "\n", + encoding="utf-8", + ) + env_a = task_a / "environment" + env_a.mkdir() + (env_a / "Dockerfile").write_text("FROM ubuntu\nWORKDIR /work\n", encoding="utf-8") + + task_b = root / "task-b" + task_b.mkdir() + (task_b / "instruction.md").write_text("solve b\n", encoding="utf-8") + (task_b / "task.toml").write_text( + """ +schema_version = "1.1" +[task] +name = "terminal-bench/other-task" +description = "demo" +[environment] +docker_image = "example/other:latest" +""".strip() + + "\n", + encoding="utf-8", + ) + env_b = task_b / "environment" + env_b.mkdir() + (env_b / "docker-compose.yaml").write_text("services: {}\n", encoding="utf-8") + + tasks = discover_tasks(root) + self.assertEqual(len(tasks), 2) + + selected = filter_tasks( + tasks, + include_patterns=["headless-*"], + exclude_patterns=[], + limit=None, + ) + self.assertEqual(len(selected), 1) + self.assertEqual(selected[0].short_name, "headless-terminal") + self.assertFalse(selected[0].has_docker_compose) + + selected = filter_tasks( + tasks, + include_patterns=[], + exclude_patterns=["other-*"], + limit=None, + ) + self.assertEqual(len(selected), 1) + self.assertEqual(selected[0].short_name, "headless-terminal") + + def test_build_host_agent_command_uses_current_interpreter(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + workspace_dir = root / "workspace" + repo_dir = root / "repo" + agent_logs_dir = root / "agent" + workspace_dir.mkdir() + repo_dir.mkdir() + agent_logs_dir.mkdir() + task = TerminalBenchTask( + task_dir=root, + name="terminal-bench/demo", + short_name="demo", + instruction="solve it", + docker_image="example/demo:latest", + agent_timeout_sec=30.0, + verifier_timeout_sec=30.0, + workdir="/workspace", + has_docker_compose=False, + ) + + cmd = build_host_agent_command( + task=task, + workspace_dir=workspace_dir, + repo_dir=repo_dir, + agent_logs_dir=agent_logs_dir, + ) + + self.assertIn(" -m src.main agent ", cmd) + self.assertIn("instruction=$(cat", cmd) + self.assertNotIn("claw-code-agent agent", cmd) + + +if __name__ == "__main__": + unittest.main()