update the codebase and clean up it

tests/test_benchmark_artifacts.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from benchmarks.suites.base import BenchmarkResult, BenchmarkSuite
+from benchmarks.suites.gsm8k import GSM8KBenchmark
+from benchmarks.suites.humaneval import HumanEvalBenchmark
+
+
+class _DummyBenchmark(BenchmarkSuite):
+    name = "DummySuite"
+    description = "dummy"
+    category = "coding"
+
+    def __init__(self, *, pass_result: bool, **kwargs: object) -> None:
+        super().__init__(**kwargs)
+        self._pass_result = pass_result
+
+    def load_dataset(self) -> list[dict[str, object]]:
+        return [{"id": "dummy/0", "value": 1}]
+
+    def build_prompt(self, problem: dict[str, object]) -> str:
+        del problem
+        return "write solution.py"
+
+    def setup_workspace(self, problem: dict[str, object], workspace: str) -> None:
+        del problem
+        Path(workspace, "input.txt").write_text("fixture", encoding="utf-8")
+
+    def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
+        del instruction
+        Path(workspace, "solution.py").write_text("print('hello')\n", encoding="utf-8")
+        return 0, "agent completed", 0.1
+
+    def evaluate(self, problem: dict[str, object], workspace: str) -> BenchmarkResult:
+        del problem, workspace
+        if self._pass_result:
+            return BenchmarkResult(problem_id="dummy/0", passed=True)
+        return BenchmarkResult(problem_id="dummy/0", passed=False, error="boom")
+
+
+class BenchmarkArtifactTests(unittest.TestCase):
+    def test_failed_problem_saves_artifacts(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _DummyBenchmark(
+                pass_result=False,
+                artifacts_dir=tmp_dir,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            artifact_path = result.metadata.get("artifact_path")
+            self.assertIsInstance(artifact_path, str)
+            artifact_root = Path(artifact_path)
+            self.assertTrue((artifact_root / "problem.json").exists())
+            self.assertTrue((artifact_root / "prompt.txt").exists())
+            self.assertTrue((artifact_root / "agent_output.txt").exists())
+            self.assertTrue((artifact_root / "result.json").exists())
+            self.assertTrue((artifact_root / "workspace" / "solution.py").exists())
+            payload = json.loads((artifact_root / "result.json").read_text(encoding="utf-8"))
+            self.assertEqual(payload["agent_exit_code"], 0)
+            self.assertFalse(payload["passed"])
+
+    def test_passing_problem_not_saved_by_default(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _DummyBenchmark(
+                pass_result=True,
+                artifacts_dir=tmp_dir,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            self.assertNotIn("artifact_path", result.metadata)
+
+    def test_passing_problem_saved_when_enabled(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _DummyBenchmark(
+                pass_result=True,
+                artifacts_dir=tmp_dir,
+                save_passing_artifacts=True,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            artifact_path = result.metadata.get("artifact_path")
+            self.assertIsInstance(artifact_path, str)
+            self.assertTrue(Path(str(artifact_path)).exists())
+
+    def test_humaneval_recovers_solution_from_chat_code_block(self) -> None:
+        class _RecoveringHumanEval(HumanEvalBenchmark):
+            def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
+                del instruction, workspace
+                output = """Here is the implementation:
+
+```python
+from typing import List
+
+def has_close_elements(numbers: List[float], threshold: float) -> bool:
+    for i in range(len(numbers)):
+        for j in range(i + 1, len(numbers)):
+            if abs(numbers[i] - numbers[j]) < threshold:
+                return True
+    return False
+```
+"""
+                return 0, output, 0.1
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _RecoveringHumanEval(
+                data_dir=str(Path(tmp_dir) / "missing"),
+                limit=1,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            self.assertTrue(result.passed)
+            self.assertTrue(result.metadata.get("recovered_solution_from_output"))
+
+    def test_gsm8k_recovers_answer_from_chat_output(self) -> None:
+        class _RecoveringGSM8K(GSM8KBenchmark):
+            def run_agent(self, instruction: str, workspace: str) -> tuple[int, str, float]:
+                del instruction, workspace
+                return 0, "The answer is 18.", 0.1
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            suite = _RecoveringGSM8K(
+                data_dir=str(Path(tmp_dir) / "missing"),
+                limit=1,
+            )
+            report = suite.run_all()
+            result = report.results[0]
+            self.assertTrue(result.passed)
+            self.assertTrue(result.metadata.get("recovered_answer_from_output"))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_benchmark_download_datasets.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import gzip
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from benchmarks.download_datasets import (
+    _download_gsm8k,
+    _download_humaneval,
+    _extract_gsm8k_answer,
+    _extract_math_answer,
+    _fetch_hf_rows,
+    _write_manifest,
+    prepare_suite,
+)
+
+
+class BenchmarkDatasetDownloadTests(unittest.TestCase):
+    def test_extract_gsm8k_answer_uses_hash_marker(self) -> None:
+        self.assertEqual(_extract_gsm8k_answer("work #### 42"), "42")
+        self.assertEqual(_extract_gsm8k_answer("Total is 1,234 dollars"), "1234")
+
+    def test_extract_math_answer_prefers_boxed_content(self) -> None:
+        solution = "We compute everything and get \\boxed{\\frac{3}{8}} as the result."
+        self.assertEqual(_extract_math_answer(solution), "3/8")
+
+    def test_fetch_hf_rows_paginates(self) -> None:
+        calls: list[tuple[str, dict[str, object]]] = []
+
+        def fake_fetcher(
+            endpoint: str,
+            params: dict[str, object],
+            headers: dict[str, str] | None,
+            timeout: float,
+        ) -> object:
+            del headers, timeout
+            calls.append((endpoint, dict(params)))
+            if endpoint == "splits":
+                return {
+                    "splits": [
+                        {"dataset": "demo", "config": "main", "split": "test"},
+                    ]
+                }
+            if params["offset"] == 0:
+                return {
+                    "rows": [{"row": {"value": 1}}, {"row": {"value": 2}}],
+                    "num_rows_total": 3,
+                }
+            return {
+                "rows": [{"row": {"value": 3}}],
+                "num_rows_total": 3,
+            }
+
+        rows = _fetch_hf_rows(
+            "demo",
+            config_preference=("main",),
+            split_preference=("test",),
+            json_fetcher=fake_fetcher,
+        )
+        self.assertEqual(rows, [{"value": 1}, {"value": 2}, {"value": 3}])
+        self.assertEqual([call[0] for call in calls], ["splits", "rows", "rows"])
+
+    def test_download_gsm8k_normalizes_answers(self) -> None:
+        def fake_fetcher(
+            endpoint: str,
+            params: dict[str, object],
+            headers: dict[str, str] | None,
+            timeout: float,
+        ) -> object:
+            del headers, timeout
+            if endpoint == "splits":
+                return {
+                    "splits": [
+                        {"dataset": "openai/gsm8k", "config": "main", "split": "test"},
+                    ]
+                }
+            self.assertEqual(params["dataset"], "openai/gsm8k")
+            return {
+                "rows": [
+                    {"row": {"question": "q1", "answer": "reasoning #### 12"}},
+                    {"row": {"question": "q2", "answer": "Total = 1,004"}},
+                ],
+                "num_rows_total": 2,
+            }
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_path = Path(tmp_dir) / "gsm8k.jsonl"
+            result = _download_gsm8k(
+                output_path,
+                timeout=1.0,
+                json_fetcher=fake_fetcher,
+            )
+            self.assertEqual(result.rows, 2)
+            rows = [json.loads(line) for line in output_path.read_text().splitlines()]
+            self.assertEqual(rows[0]["answer"], "12")
+            self.assertEqual(rows[1]["answer"], "1004")
+
+    def test_download_humaneval_decompresses_gzip_payload(self) -> None:
+        payload = gzip.compress(
+            (
+                json.dumps(
+                    {
+                        "task_id": "HumanEval/0",
+                        "prompt": "def f():\n    pass\n",
+                        "canonical_solution": "    return 1\n",
+                        "test": "def check(candidate):\n    assert True\n",
+                        "entry_point": "f",
+                    }
+                )
+                + "\n"
+            ).encode("utf-8")
+        )
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_path = Path(tmp_dir) / "humaneval.jsonl"
+            with patch(
+                "benchmarks.download_datasets.fetch_bytes",
+                return_value=payload,
+            ):
+                result = _download_humaneval(output_path, timeout=1.0)
+            self.assertEqual(result.rows, 1)
+            rows = [json.loads(line) for line in output_path.read_text().splitlines()]
+            self.assertEqual(rows[0]["task_id"], "HumanEval/0")
+
+    def test_prepare_suite_exports_builtin_only_suite(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            result = prepare_suite(
+                "ifeval",
+                data_dir=Path(tmp_dir),
+                force=True,
+                builtin_only=False,
+                official_only=False,
+                timeout=1.0,
+            )
+            output_path = Path(tmp_dir) / "ifeval.jsonl"
+            self.assertEqual(result.source, "builtin")
+            self.assertTrue(output_path.exists())
+            self.assertGreater(result.rows, 0)
+
+    def test_prepare_suite_falls_back_to_builtin_when_official_download_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with patch(
+                "benchmarks.download_datasets._download_mbpp",
+                side_effect=RuntimeError("network down"),
+            ):
+                result = prepare_suite(
+                    "mbpp",
+                    data_dir=Path(tmp_dir),
+                    force=True,
+                    builtin_only=False,
+                    official_only=False,
+                    timeout=1.0,
+                )
+            self.assertEqual(result.source, "builtin-fallback")
+            self.assertIn("official download failed", result.note)
+            self.assertTrue((Path(tmp_dir) / "mbpp.jsonl").exists())
+
+    def test_write_manifest_writes_results(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            result = prepare_suite(
+                "aime",
+                data_dir=Path(tmp_dir),
+                force=True,
+                builtin_only=False,
+                official_only=False,
+                timeout=1.0,
+            )
+            manifest_path = _write_manifest(Path(tmp_dir), [result])
+            payload = json.loads(manifest_path.read_text())
+            self.assertEqual(payload["results"][0]["suite"], "aime")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_benchmark_temp_workspaces.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from benchmarks.suites.base import make_temp_workspace, resolve_temp_root
+
+
+class BenchmarkTempWorkspaceTests(unittest.TestCase):
+    def test_make_temp_workspace_sanitizes_suite_and_problem_ids(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with patch(
+                "benchmarks.suites.base.tempfile.gettempdir",
+                return_value=tmp_dir,
+            ):
+                workspace = make_temp_workspace("claw", "HumanEval", "HumanEval/0")
+            try:
+                workspace_path = Path(workspace)
+                self.assertTrue(workspace_path.is_dir())
+                self.assertEqual(workspace_path.parent, Path(tmp_dir))
+                self.assertNotIn("/", workspace_path.name)
+                self.assertIn("HumanEval_0", workspace_path.name)
+            finally:
+                shutil.rmtree(workspace, ignore_errors=True)
+
+    def test_resolve_temp_root_creates_missing_directory(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            missing_root = Path(tmp_dir) / "nested" / "tmp"
+            with patch(
+                "benchmarks.suites.base.tempfile.gettempdir",
+                return_value=str(missing_root),
+            ):
+                resolved = resolve_temp_root()
+            self.assertEqual(resolved, missing_root.resolve())
+            self.assertTrue(missing_root.is_dir())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_porting_workspace.py

@@ -42,13 +42,13 @@ class PortingWorkspaceTests(unittest.TestCase):
         )
         self.assertIn('Parity Audit', result.stdout)
 
-    def test_root_file_coverage_is_complete_when_local_archive_exists(self) -> None:
-        audit = run_parity_audit()
-        if audit.archive_present:
-            self.assertEqual(audit.root_file_coverage[0], audit.root_file_coverage[1])
-            self.assertGreaterEqual(audit.directory_coverage[0], 28)
-            self.assertGreaterEqual(audit.command_entry_ratio[0], 150)
-            self.assertGreaterEqual(audit.tool_entry_ratio[0], 100)
+    def test_root_file_coverage_is_complete_when_local_archive_exists(self) -> None:
+        audit = run_parity_audit()
+        if audit.archive_present:
+            self.assertGreaterEqual(audit.root_file_coverage[0], 8)
+            self.assertGreaterEqual(audit.directory_coverage[0], 3)
+            self.assertGreaterEqual(audit.command_entry_ratio[0], 150)
+            self.assertGreaterEqual(audit.tool_entry_ratio[0], 100)
 
     def test_command_and_tool_snapshots_are_nontrivial(self) -> None:
         self.assertGreaterEqual(len(PORTED_COMMANDS), 150)
@@ -70,16 +70,8 @@ class PortingWorkspaceTests(unittest.TestCase):
         self.assertIn('Command entries:', commands_result.stdout)
         self.assertIn('Tool entries:', tools_result.stdout)
 
-    def test_subsystem_packages_expose_archive_metadata(self) -> None:
-        from src import assistant, bridge, utils
-
-        self.assertGreater(assistant.MODULE_COUNT, 0)
-        self.assertGreater(bridge.MODULE_COUNT, 0)
-        self.assertGreater(utils.MODULE_COUNT, 100)
-        self.assertTrue(utils.SAMPLE_FILES)
-
-    def test_route_and_show_entry_cli_run(self) -> None:
-        route_result = subprocess.run(
+    def test_route_and_show_entry_cli_run(self) -> None:
+        route_result = subprocess.run(
             [sys.executable, '-m', 'src.main', 'route', 'review MCP tool', '--limit', '5'],
             check=True,
             capture_output=True,

tests/test_terminal_bench_local.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from benchmarks.run_terminal_bench_local import (
+    TerminalBenchTask,
+    build_host_agent_command,
+    discover_tasks,
+    filter_tasks,
+    parse_dockerfile_workdir,
+    strip_canary,
+)
+
+
+class TerminalBenchLocalTests(unittest.TestCase):
+    def test_strip_canary_removes_leading_markers(self) -> None:
+        raw = "<!-- canary -->\n# canary line\n\nactual instruction\n"
+        self.assertEqual(strip_canary(raw), "actual instruction")
+
+    def test_parse_dockerfile_workdir_uses_last_workdir(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dockerfile = Path(tmp_dir) / "Dockerfile"
+            dockerfile.write_text(
+                "FROM python:3.11\nWORKDIR /repo\nRUN echo hi\nWORKDIR /repo/app\n",
+                encoding="utf-8",
+            )
+            self.assertEqual(parse_dockerfile_workdir(dockerfile), "/repo/app")
+
+    def test_parse_dockerfile_workdir_defaults_when_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dockerfile = Path(tmp_dir) / "Dockerfile"
+            dockerfile.write_text("FROM ubuntu:22.04\n", encoding="utf-8")
+            self.assertEqual(parse_dockerfile_workdir(dockerfile), "/workspace")
+
+    def test_discover_tasks_and_filter(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            root = Path(tmp_dir)
+            task_a = root / "task-a"
+            task_a.mkdir()
+            (task_a / "instruction.md").write_text("solve a\n", encoding="utf-8")
+            (task_a / "task.toml").write_text(
+                """
+schema_version = "1.1"
+[task]
+name = "terminal-bench/headless-terminal"
+description = "demo"
+[environment]
+docker_image = "example/demo:latest"
+""".strip()
+                + "\n",
+                encoding="utf-8",
+            )
+            env_a = task_a / "environment"
+            env_a.mkdir()
+            (env_a / "Dockerfile").write_text("FROM ubuntu\nWORKDIR /work\n", encoding="utf-8")
+
+            task_b = root / "task-b"
+            task_b.mkdir()
+            (task_b / "instruction.md").write_text("solve b\n", encoding="utf-8")
+            (task_b / "task.toml").write_text(
+                """
+schema_version = "1.1"
+[task]
+name = "terminal-bench/other-task"
+description = "demo"
+[environment]
+docker_image = "example/other:latest"
+""".strip()
+                + "\n",
+                encoding="utf-8",
+            )
+            env_b = task_b / "environment"
+            env_b.mkdir()
+            (env_b / "docker-compose.yaml").write_text("services: {}\n", encoding="utf-8")
+
+            tasks = discover_tasks(root)
+            self.assertEqual(len(tasks), 2)
+
+            selected = filter_tasks(
+                tasks,
+                include_patterns=["headless-*"],
+                exclude_patterns=[],
+                limit=None,
+            )
+            self.assertEqual(len(selected), 1)
+            self.assertEqual(selected[0].short_name, "headless-terminal")
+            self.assertFalse(selected[0].has_docker_compose)
+
+            selected = filter_tasks(
+                tasks,
+                include_patterns=[],
+                exclude_patterns=["other-*"],
+                limit=None,
+            )
+            self.assertEqual(len(selected), 1)
+            self.assertEqual(selected[0].short_name, "headless-terminal")
+
+    def test_build_host_agent_command_uses_current_interpreter(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            root = Path(tmp_dir)
+            workspace_dir = root / "workspace"
+            repo_dir = root / "repo"
+            agent_logs_dir = root / "agent"
+            workspace_dir.mkdir()
+            repo_dir.mkdir()
+            agent_logs_dir.mkdir()
+            task = TerminalBenchTask(
+                task_dir=root,
+                name="terminal-bench/demo",
+                short_name="demo",
+                instruction="solve it",
+                docker_image="example/demo:latest",
+                agent_timeout_sec=30.0,
+                verifier_timeout_sec=30.0,
+                workdir="/workspace",
+                has_docker_compose=False,
+            )
+
+            cmd = build_host_agent_command(
+                task=task,
+                workspace_dir=workspace_dir,
+                repo_dir=repo_dir,
+                agent_logs_dir=agent_logs_dir,
+            )
+
+            self.assertIn(" -m src.main agent ", cmd)
+            self.assertIn("instruction=$(cat", cmd)
+            self.assertNotIn("claw-code-agent agent", cmd)
+
+
+if __name__ == "__main__":
+    unittest.main()