copilot-swe-agent[bot]
231b977b92
Implements HumanEval, MBPP, SWE-Bench, Aider, LiveCodeBench (coding), MATH, GSM8K, AIME (math), and IFEval, BFCL (instruction following). Each suite includes built-in problem subsets (108 total) and supports loading full datasets from JSONL files. Includes comprehensive README with all commands. Agent-Logs-Url: https://github.com/HarnessLab/claw-code-agent/sessions/6890e3d0-3058-4b1f-b7e5-27171c079c62 Co-authored-by: abdoelsayed2016 <27821589+abdoelsayed2016@users.noreply.github.com>
212 lines
8.1 KiB
Python
212 lines
8.1 KiB
Python
"""
|
|
SWE-Bench benchmark suite.
|
|
|
|
SWE-Bench tests whether an agent can resolve real GitHub issues.
|
|
Each problem provides a repository, an issue description, and a patch
|
|
that should make the failing tests pass.
|
|
|
|
Paper: https://arxiv.org/abs/2310.06770
|
|
Dataset: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import textwrap
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .base import BenchmarkResult, BenchmarkSuite
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Built-in mini dataset — simplified SWE-Bench-style problems
|
|
# These are self-contained (no actual repo cloning needed).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_BUILTIN_PROBLEMS: list[dict[str, Any]] = [
|
|
{
|
|
"instance_id": "swe-mini-001",
|
|
"problem_statement": (
|
|
"The `StringFormatter.format_title` method should capitalize the first letter "
|
|
"of each word in the input string. Currently it lowercases everything instead.\n\n"
|
|
"Expected: format_title('hello world') => 'Hello World'\n"
|
|
"Actual: format_title('hello world') => 'hello world'"
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > formatter.py << 'PYEOF'
|
|
class StringFormatter:
|
|
def format_title(self, text):
|
|
return text.lower()
|
|
PYEOF
|
|
cat > test_formatter.py << 'PYEOF'
|
|
from formatter import StringFormatter
|
|
f = StringFormatter()
|
|
assert f.format_title('hello world') == 'Hello World'
|
|
assert f.format_title('PYTHON IS FUN') == 'Python Is Fun'
|
|
assert f.format_title('already Good') == 'Already Good'
|
|
print("ALL_TESTS_PASSED")
|
|
PYEOF
|
|
"""),
|
|
"test_cmd": "python3 test_formatter.py",
|
|
},
|
|
{
|
|
"instance_id": "swe-mini-002",
|
|
"problem_statement": (
|
|
"The `DataValidator.validate_email` function returns True for emails "
|
|
"that don't have a domain extension (e.g., 'user@domain' without .com). "
|
|
"It should require at least one dot after the @ sign.\n\n"
|
|
"Expected: validate_email('user@domain') => False\n"
|
|
"Actual: validate_email('user@domain') => True"
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > validator.py << 'PYEOF'
|
|
class DataValidator:
|
|
def validate_email(self, email):
|
|
return '@' in email
|
|
PYEOF
|
|
cat > test_validator.py << 'PYEOF'
|
|
from validator import DataValidator
|
|
v = DataValidator()
|
|
assert v.validate_email('user@example.com') == True
|
|
assert v.validate_email('bad') == False
|
|
assert v.validate_email('user@domain') == False
|
|
assert v.validate_email('a@b.c') == True
|
|
print("ALL_TESTS_PASSED")
|
|
PYEOF
|
|
"""),
|
|
"test_cmd": "python3 test_validator.py",
|
|
},
|
|
{
|
|
"instance_id": "swe-mini-003",
|
|
"problem_statement": (
|
|
"The `FileProcessor.count_lines` method crashes with an unhandled "
|
|
"exception when the file doesn't exist. It should return -1 for missing files.\n\n"
|
|
"Expected: count_lines('nonexistent.txt') => -1\n"
|
|
"Actual: FileNotFoundError is raised"
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > processor.py << 'PYEOF'
|
|
class FileProcessor:
|
|
def count_lines(self, filepath):
|
|
with open(filepath) as f:
|
|
return len(f.readlines())
|
|
PYEOF
|
|
echo -e "line1\\nline2\\nline3" > sample.txt
|
|
cat > test_processor.py << 'PYEOF'
|
|
from processor import FileProcessor
|
|
p = FileProcessor()
|
|
assert p.count_lines('sample.txt') == 3
|
|
assert p.count_lines('nonexistent.txt') == -1
|
|
print("ALL_TESTS_PASSED")
|
|
PYEOF
|
|
"""),
|
|
"test_cmd": "python3 test_processor.py",
|
|
},
|
|
{
|
|
"instance_id": "swe-mini-004",
|
|
"problem_statement": (
|
|
"The `MathHelper.factorial` function gives wrong results for n=0. "
|
|
"factorial(0) should return 1, but it currently returns 0.\n\n"
|
|
"Expected: factorial(0) => 1\n"
|
|
"Actual: factorial(0) => 0"
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > mathhelper.py << 'PYEOF'
|
|
class MathHelper:
|
|
def factorial(self, n):
|
|
result = 0
|
|
for i in range(1, n + 1):
|
|
result *= i
|
|
return result
|
|
PYEOF
|
|
cat > test_math.py << 'PYEOF'
|
|
from mathhelper import MathHelper
|
|
m = MathHelper()
|
|
assert m.factorial(0) == 1
|
|
assert m.factorial(1) == 1
|
|
assert m.factorial(5) == 120
|
|
assert m.factorial(10) == 3628800
|
|
print("ALL_TESTS_PASSED")
|
|
PYEOF
|
|
"""),
|
|
"test_cmd": "python3 test_math.py",
|
|
},
|
|
{
|
|
"instance_id": "swe-mini-005",
|
|
"problem_statement": (
|
|
"The `ListUtils.unique` method should return unique elements in the "
|
|
"order they first appear, but it currently sorts them alphabetically.\n\n"
|
|
"Expected: unique(['b', 'a', 'b', 'c', 'a']) => ['b', 'a', 'c']\n"
|
|
"Actual: unique(['b', 'a', 'b', 'c', 'a']) => ['a', 'b', 'c']"
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > listutils.py << 'PYEOF'
|
|
class ListUtils:
|
|
def unique(self, items):
|
|
return sorted(set(items))
|
|
PYEOF
|
|
cat > test_list.py << 'PYEOF'
|
|
from listutils import ListUtils
|
|
u = ListUtils()
|
|
assert u.unique(['b', 'a', 'b', 'c', 'a']) == ['b', 'a', 'c']
|
|
assert u.unique([3, 1, 2, 1, 3]) == [3, 1, 2]
|
|
assert u.unique([]) == []
|
|
assert u.unique([1]) == [1]
|
|
print("ALL_TESTS_PASSED")
|
|
PYEOF
|
|
"""),
|
|
"test_cmd": "python3 test_list.py",
|
|
},
|
|
]
|
|
|
|
|
|
class SWEBenchBenchmark(BenchmarkSuite):
|
|
"""SWE-Bench: resolve real-world GitHub issues."""
|
|
|
|
name = "SWE-Bench"
|
|
description = "Resolve GitHub issues by editing source code"
|
|
category = "coding"
|
|
|
|
def load_dataset(self) -> list[dict[str, Any]]:
|
|
jsonl_path = Path(self.data_dir) / "swe_bench.jsonl"
|
|
if jsonl_path.exists():
|
|
problems: list[dict[str, Any]] = []
|
|
with open(jsonl_path) as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if line:
|
|
problems.append(json.loads(line))
|
|
if self.verbose:
|
|
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
|
return problems
|
|
|
|
if self.verbose:
|
|
print(f" {jsonl_path} not found — using built-in 5-problem subset")
|
|
return list(_BUILTIN_PROBLEMS)
|
|
|
|
def build_prompt(self, problem: dict[str, Any]) -> str:
|
|
stmt = problem["problem_statement"]
|
|
return (
|
|
f"You are a software engineer fixing a bug. Here is the issue:\n\n"
|
|
f"{stmt}\n\n"
|
|
f"Read the code in the workspace, find the bug, and fix it. "
|
|
f"Do not modify the test files."
|
|
)
|
|
|
|
def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
|
|
setup = problem.get("setup_code", "")
|
|
if setup:
|
|
self._run_shell(setup, cwd=workspace, timeout=30.0)
|
|
|
|
def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
|
|
pid = problem.get("instance_id", "unknown")
|
|
test_cmd = problem.get("test_cmd", "echo FAIL && exit 1")
|
|
code, output = self._run_shell(test_cmd, cwd=workspace, timeout=30.0)
|
|
passed = code == 0 and "ALL_TESTS_PASSED" in output
|
|
return BenchmarkResult(
|
|
problem_id=pid, passed=passed, actual=output[:500],
|
|
error="" if passed else output[:500],
|
|
)
|