copilot-swe-agent[bot]
231b977b92
Implements HumanEval, MBPP, SWE-Bench, Aider, LiveCodeBench (coding), MATH, GSM8K, AIME (math), and IFEval, BFCL (instruction following). Each suite includes built-in problem subsets (108 total) and supports loading full datasets from JSONL files. Includes comprehensive README with all commands. Agent-Logs-Url: https://github.com/HarnessLab/claw-code-agent/sessions/6890e3d0-3058-4b1f-b7e5-27171c079c62 Co-authored-by: abdoelsayed2016 <27821589+abdoelsayed2016@users.noreply.github.com>
301 lines
10 KiB
Python
301 lines
10 KiB
Python
"""
|
|
Aider benchmark suite.
|
|
|
|
Aider's code editing benchmark measures how well an agent can apply
|
|
specified edits to existing codebases — refactoring, adding features,
|
|
and fixing bugs based on natural language instructions.
|
|
|
|
Reference: https://aider.chat/docs/leaderboards/
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import textwrap
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .base import BenchmarkResult, BenchmarkSuite
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Built-in mini dataset — Aider-style edit tasks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_BUILTIN_PROBLEMS: list[dict[str, Any]] = [
|
|
{
|
|
"id": "aider-001",
|
|
"instruction": "Add a `__len__` method to the `TaskList` class that returns the number of tasks.",
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > tasks.py << 'PYEOF'
|
|
class TaskList:
|
|
def __init__(self):
|
|
self._tasks = []
|
|
|
|
def add(self, task):
|
|
self._tasks.append(task)
|
|
|
|
def get_all(self):
|
|
return list(self._tasks)
|
|
PYEOF
|
|
"""),
|
|
"test_code": textwrap.dedent("""\
|
|
from tasks import TaskList
|
|
t = TaskList()
|
|
assert len(t) == 0
|
|
t.add("task1")
|
|
t.add("task2")
|
|
assert len(t) == 2
|
|
# Ensure original methods still work
|
|
assert t.get_all() == ["task1", "task2"]
|
|
print("ALL_TESTS_PASSED")
|
|
"""),
|
|
},
|
|
{
|
|
"id": "aider-002",
|
|
"instruction": (
|
|
"Refactor the `process_data` function to use a list comprehension "
|
|
"instead of the for loop. The behavior must remain identical."
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > processor.py << 'PYEOF'
|
|
def process_data(items):
|
|
result = []
|
|
for item in items:
|
|
if item > 0:
|
|
result.append(item * 2)
|
|
return result
|
|
PYEOF
|
|
"""),
|
|
"test_code": textwrap.dedent("""\
|
|
import inspect
|
|
from processor import process_data
|
|
assert process_data([1, -2, 3, -4, 5]) == [2, 6, 10]
|
|
assert process_data([]) == []
|
|
assert process_data([-1, -2]) == []
|
|
# Check that a list comprehension is used
|
|
src = inspect.getsource(process_data)
|
|
assert 'for' in src and '[' in src, "Should use list comprehension"
|
|
assert src.count('for') <= 2, "Should not have a separate for loop"
|
|
print("ALL_TESTS_PASSED")
|
|
"""),
|
|
},
|
|
{
|
|
"id": "aider-003",
|
|
"instruction": (
|
|
"Add error handling to the `divide` function so it raises a "
|
|
"`ValueError` with message 'Division by zero' when b is 0, "
|
|
"instead of letting ZeroDivisionError propagate."
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > mathops.py << 'PYEOF'
|
|
def divide(a, b):
|
|
return a / b
|
|
PYEOF
|
|
"""),
|
|
"test_code": textwrap.dedent("""\
|
|
from mathops import divide
|
|
assert divide(10, 2) == 5.0
|
|
assert divide(7, 2) == 3.5
|
|
try:
|
|
divide(1, 0)
|
|
assert False, "Should have raised ValueError"
|
|
except ValueError as e:
|
|
assert str(e) == 'Division by zero'
|
|
except ZeroDivisionError:
|
|
assert False, "Should raise ValueError, not ZeroDivisionError"
|
|
print("ALL_TESTS_PASSED")
|
|
"""),
|
|
},
|
|
{
|
|
"id": "aider-004",
|
|
"instruction": (
|
|
"Add a `to_dict` method to the `Config` class that returns a dictionary "
|
|
"of all configuration key-value pairs."
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > config.py << 'PYEOF'
|
|
class Config:
|
|
def __init__(self):
|
|
self._data = {}
|
|
|
|
def set(self, key, value):
|
|
self._data[key] = value
|
|
|
|
def get(self, key, default=None):
|
|
return self._data.get(key, default)
|
|
PYEOF
|
|
"""),
|
|
"test_code": textwrap.dedent("""\
|
|
from config import Config
|
|
c = Config()
|
|
assert c.to_dict() == {}
|
|
c.set('host', 'localhost')
|
|
c.set('port', 8080)
|
|
d = c.to_dict()
|
|
assert d == {'host': 'localhost', 'port': 8080}
|
|
# Ensure it's a copy, not a reference
|
|
d['new'] = 'value'
|
|
assert 'new' not in c.to_dict()
|
|
print("ALL_TESTS_PASSED")
|
|
"""),
|
|
},
|
|
{
|
|
"id": "aider-005",
|
|
"instruction": (
|
|
"Add a `reverse` method to the `LinkedList` class that reverses the "
|
|
"list in-place."
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > linkedlist.py << 'PYEOF'
|
|
class Node:
|
|
def __init__(self, val, next=None):
|
|
self.val = val
|
|
self.next = next
|
|
|
|
class LinkedList:
|
|
def __init__(self):
|
|
self.head = None
|
|
|
|
def append(self, val):
|
|
if not self.head:
|
|
self.head = Node(val)
|
|
return
|
|
curr = self.head
|
|
while curr.next:
|
|
curr = curr.next
|
|
curr.next = Node(val)
|
|
|
|
def to_list(self):
|
|
result = []
|
|
curr = self.head
|
|
while curr:
|
|
result.append(curr.val)
|
|
curr = curr.next
|
|
return result
|
|
PYEOF
|
|
"""),
|
|
"test_code": textwrap.dedent("""\
|
|
from linkedlist import LinkedList
|
|
ll = LinkedList()
|
|
ll.append(1)
|
|
ll.append(2)
|
|
ll.append(3)
|
|
ll.reverse()
|
|
assert ll.to_list() == [3, 2, 1]
|
|
# Test single element
|
|
ll2 = LinkedList()
|
|
ll2.append(42)
|
|
ll2.reverse()
|
|
assert ll2.to_list() == [42]
|
|
# Test empty
|
|
ll3 = LinkedList()
|
|
ll3.reverse()
|
|
assert ll3.to_list() == []
|
|
print("ALL_TESTS_PASSED")
|
|
"""),
|
|
},
|
|
{
|
|
"id": "aider-006",
|
|
"instruction": (
|
|
"Convert the `UserStore` class to use a context manager pattern. "
|
|
"Add `__enter__` and `__exit__` methods. `__enter__` should return self, "
|
|
"and `__exit__` should call the existing `close` method."
|
|
),
|
|
"setup_code": textwrap.dedent("""\
|
|
cat > userstore.py << 'PYEOF'
|
|
class UserStore:
|
|
def __init__(self):
|
|
self._users = {}
|
|
self._closed = False
|
|
|
|
def add(self, name, email):
|
|
self._users[name] = email
|
|
|
|
def get(self, name):
|
|
return self._users.get(name)
|
|
|
|
def close(self):
|
|
self._closed = True
|
|
|
|
@property
|
|
def is_closed(self):
|
|
return self._closed
|
|
PYEOF
|
|
"""),
|
|
"test_code": textwrap.dedent("""\
|
|
from userstore import UserStore
|
|
with UserStore() as store:
|
|
store.add("alice", "alice@example.com")
|
|
assert store.get("alice") == "alice@example.com"
|
|
assert not store.is_closed
|
|
assert store.is_closed
|
|
# Test exception in with block still calls close
|
|
try:
|
|
with UserStore() as store2:
|
|
store2.add("bob", "bob@example.com")
|
|
raise ValueError("test error")
|
|
except ValueError:
|
|
pass
|
|
assert store2.is_closed
|
|
print("ALL_TESTS_PASSED")
|
|
"""),
|
|
},
|
|
]
|
|
|
|
|
|
class AiderBenchmark(BenchmarkSuite):
|
|
"""Aider: code editing benchmark."""
|
|
|
|
name = "Aider"
|
|
description = "Code editing and refactoring tasks"
|
|
category = "coding"
|
|
|
|
def load_dataset(self) -> list[dict[str, Any]]:
|
|
jsonl_path = Path(self.data_dir) / "aider.jsonl"
|
|
if jsonl_path.exists():
|
|
problems: list[dict[str, Any]] = []
|
|
with open(jsonl_path) as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if line:
|
|
problems.append(json.loads(line))
|
|
if self.verbose:
|
|
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
|
return problems
|
|
|
|
if self.verbose:
|
|
print(f" {jsonl_path} not found — using built-in 6-problem subset")
|
|
return list(_BUILTIN_PROBLEMS)
|
|
|
|
def build_prompt(self, problem: dict[str, Any]) -> str:
|
|
return (
|
|
f"Edit the code in the workspace to satisfy this requirement:\n\n"
|
|
f"{problem['instruction']}\n\n"
|
|
f"Read the existing file(s), make the minimal changes needed, "
|
|
f"and save the updated file(s)."
|
|
)
|
|
|
|
def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
|
|
setup = problem.get("setup_code", "")
|
|
if setup:
|
|
self._run_shell(setup, cwd=workspace, timeout=30.0)
|
|
|
|
# Write test file
|
|
test_code = problem.get("test_code", "")
|
|
if test_code:
|
|
with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
|
|
fh.write(test_code)
|
|
|
|
def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
|
|
pid = problem.get("id", "unknown")
|
|
code, output = self._run_shell(
|
|
f"{sys.executable} test_harness.py", cwd=workspace, timeout=30.0
|
|
)
|
|
passed = code == 0 and "ALL_TESTS_PASSED" in output
|
|
return BenchmarkResult(
|
|
problem_id=pid, passed=passed, actual=output[:500],
|
|
error="" if passed else output[:500],
|
|
)
|