Add 10 standard evaluation benchmark suites with CLI runner and README
Implements HumanEval, MBPP, SWE-Bench, Aider, LiveCodeBench (coding), MATH, GSM8K, AIME (math), and IFEval, BFCL (instruction following). Each suite includes built-in problem subsets (108 total) and supports loading full datasets from JSONL files. Includes comprehensive README with all commands. Agent-Logs-Url: https://github.com/HarnessLab/claw-code-agent/sessions/6890e3d0-3058-4b1f-b7e5-27171c079c62 Co-authored-by: abdoelsayed2016 <27821589+abdoelsayed2016@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
committed by
GitHub
GitHub
parent
3e32154618
commit
231b977b92
@@ -0,0 +1,300 @@
|
||||
"""
|
||||
Aider benchmark suite.
|
||||
|
||||
Aider's code editing benchmark measures how well an agent can apply
|
||||
specified edits to existing codebases — refactoring, adding features,
|
||||
and fixing bugs based on natural language instructions.
|
||||
|
||||
Reference: https://aider.chat/docs/leaderboards/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .base import BenchmarkResult, BenchmarkSuite
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in mini dataset — Aider-style edit tasks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BUILTIN_PROBLEMS: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "aider-001",
|
||||
"instruction": "Add a `__len__` method to the `TaskList` class that returns the number of tasks.",
|
||||
"setup_code": textwrap.dedent("""\
|
||||
cat > tasks.py << 'PYEOF'
|
||||
class TaskList:
|
||||
def __init__(self):
|
||||
self._tasks = []
|
||||
|
||||
def add(self, task):
|
||||
self._tasks.append(task)
|
||||
|
||||
def get_all(self):
|
||||
return list(self._tasks)
|
||||
PYEOF
|
||||
"""),
|
||||
"test_code": textwrap.dedent("""\
|
||||
from tasks import TaskList
|
||||
t = TaskList()
|
||||
assert len(t) == 0
|
||||
t.add("task1")
|
||||
t.add("task2")
|
||||
assert len(t) == 2
|
||||
# Ensure original methods still work
|
||||
assert t.get_all() == ["task1", "task2"]
|
||||
print("ALL_TESTS_PASSED")
|
||||
"""),
|
||||
},
|
||||
{
|
||||
"id": "aider-002",
|
||||
"instruction": (
|
||||
"Refactor the `process_data` function to use a list comprehension "
|
||||
"instead of the for loop. The behavior must remain identical."
|
||||
),
|
||||
"setup_code": textwrap.dedent("""\
|
||||
cat > processor.py << 'PYEOF'
|
||||
def process_data(items):
|
||||
result = []
|
||||
for item in items:
|
||||
if item > 0:
|
||||
result.append(item * 2)
|
||||
return result
|
||||
PYEOF
|
||||
"""),
|
||||
"test_code": textwrap.dedent("""\
|
||||
import inspect
|
||||
from processor import process_data
|
||||
assert process_data([1, -2, 3, -4, 5]) == [2, 6, 10]
|
||||
assert process_data([]) == []
|
||||
assert process_data([-1, -2]) == []
|
||||
# Check that a list comprehension is used
|
||||
src = inspect.getsource(process_data)
|
||||
assert 'for' in src and '[' in src, "Should use list comprehension"
|
||||
assert src.count('for') <= 2, "Should not have a separate for loop"
|
||||
print("ALL_TESTS_PASSED")
|
||||
"""),
|
||||
},
|
||||
{
|
||||
"id": "aider-003",
|
||||
"instruction": (
|
||||
"Add error handling to the `divide` function so it raises a "
|
||||
"`ValueError` with message 'Division by zero' when b is 0, "
|
||||
"instead of letting ZeroDivisionError propagate."
|
||||
),
|
||||
"setup_code": textwrap.dedent("""\
|
||||
cat > mathops.py << 'PYEOF'
|
||||
def divide(a, b):
|
||||
return a / b
|
||||
PYEOF
|
||||
"""),
|
||||
"test_code": textwrap.dedent("""\
|
||||
from mathops import divide
|
||||
assert divide(10, 2) == 5.0
|
||||
assert divide(7, 2) == 3.5
|
||||
try:
|
||||
divide(1, 0)
|
||||
assert False, "Should have raised ValueError"
|
||||
except ValueError as e:
|
||||
assert str(e) == 'Division by zero'
|
||||
except ZeroDivisionError:
|
||||
assert False, "Should raise ValueError, not ZeroDivisionError"
|
||||
print("ALL_TESTS_PASSED")
|
||||
"""),
|
||||
},
|
||||
{
|
||||
"id": "aider-004",
|
||||
"instruction": (
|
||||
"Add a `to_dict` method to the `Config` class that returns a dictionary "
|
||||
"of all configuration key-value pairs."
|
||||
),
|
||||
"setup_code": textwrap.dedent("""\
|
||||
cat > config.py << 'PYEOF'
|
||||
class Config:
|
||||
def __init__(self):
|
||||
self._data = {}
|
||||
|
||||
def set(self, key, value):
|
||||
self._data[key] = value
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self._data.get(key, default)
|
||||
PYEOF
|
||||
"""),
|
||||
"test_code": textwrap.dedent("""\
|
||||
from config import Config
|
||||
c = Config()
|
||||
assert c.to_dict() == {}
|
||||
c.set('host', 'localhost')
|
||||
c.set('port', 8080)
|
||||
d = c.to_dict()
|
||||
assert d == {'host': 'localhost', 'port': 8080}
|
||||
# Ensure it's a copy, not a reference
|
||||
d['new'] = 'value'
|
||||
assert 'new' not in c.to_dict()
|
||||
print("ALL_TESTS_PASSED")
|
||||
"""),
|
||||
},
|
||||
{
|
||||
"id": "aider-005",
|
||||
"instruction": (
|
||||
"Add a `reverse` method to the `LinkedList` class that reverses the "
|
||||
"list in-place."
|
||||
),
|
||||
"setup_code": textwrap.dedent("""\
|
||||
cat > linkedlist.py << 'PYEOF'
|
||||
class Node:
|
||||
def __init__(self, val, next=None):
|
||||
self.val = val
|
||||
self.next = next
|
||||
|
||||
class LinkedList:
|
||||
def __init__(self):
|
||||
self.head = None
|
||||
|
||||
def append(self, val):
|
||||
if not self.head:
|
||||
self.head = Node(val)
|
||||
return
|
||||
curr = self.head
|
||||
while curr.next:
|
||||
curr = curr.next
|
||||
curr.next = Node(val)
|
||||
|
||||
def to_list(self):
|
||||
result = []
|
||||
curr = self.head
|
||||
while curr:
|
||||
result.append(curr.val)
|
||||
curr = curr.next
|
||||
return result
|
||||
PYEOF
|
||||
"""),
|
||||
"test_code": textwrap.dedent("""\
|
||||
from linkedlist import LinkedList
|
||||
ll = LinkedList()
|
||||
ll.append(1)
|
||||
ll.append(2)
|
||||
ll.append(3)
|
||||
ll.reverse()
|
||||
assert ll.to_list() == [3, 2, 1]
|
||||
# Test single element
|
||||
ll2 = LinkedList()
|
||||
ll2.append(42)
|
||||
ll2.reverse()
|
||||
assert ll2.to_list() == [42]
|
||||
# Test empty
|
||||
ll3 = LinkedList()
|
||||
ll3.reverse()
|
||||
assert ll3.to_list() == []
|
||||
print("ALL_TESTS_PASSED")
|
||||
"""),
|
||||
},
|
||||
{
|
||||
"id": "aider-006",
|
||||
"instruction": (
|
||||
"Convert the `UserStore` class to use a context manager pattern. "
|
||||
"Add `__enter__` and `__exit__` methods. `__enter__` should return self, "
|
||||
"and `__exit__` should call the existing `close` method."
|
||||
),
|
||||
"setup_code": textwrap.dedent("""\
|
||||
cat > userstore.py << 'PYEOF'
|
||||
class UserStore:
|
||||
def __init__(self):
|
||||
self._users = {}
|
||||
self._closed = False
|
||||
|
||||
def add(self, name, email):
|
||||
self._users[name] = email
|
||||
|
||||
def get(self, name):
|
||||
return self._users.get(name)
|
||||
|
||||
def close(self):
|
||||
self._closed = True
|
||||
|
||||
@property
|
||||
def is_closed(self):
|
||||
return self._closed
|
||||
PYEOF
|
||||
"""),
|
||||
"test_code": textwrap.dedent("""\
|
||||
from userstore import UserStore
|
||||
with UserStore() as store:
|
||||
store.add("alice", "alice@example.com")
|
||||
assert store.get("alice") == "alice@example.com"
|
||||
assert not store.is_closed
|
||||
assert store.is_closed
|
||||
# Test exception in with block still calls close
|
||||
try:
|
||||
with UserStore() as store2:
|
||||
store2.add("bob", "bob@example.com")
|
||||
raise ValueError("test error")
|
||||
except ValueError:
|
||||
pass
|
||||
assert store2.is_closed
|
||||
print("ALL_TESTS_PASSED")
|
||||
"""),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class AiderBenchmark(BenchmarkSuite):
|
||||
"""Aider: code editing benchmark."""
|
||||
|
||||
name = "Aider"
|
||||
description = "Code editing and refactoring tasks"
|
||||
category = "coding"
|
||||
|
||||
def load_dataset(self) -> list[dict[str, Any]]:
|
||||
jsonl_path = Path(self.data_dir) / "aider.jsonl"
|
||||
if jsonl_path.exists():
|
||||
problems: list[dict[str, Any]] = []
|
||||
with open(jsonl_path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line:
|
||||
problems.append(json.loads(line))
|
||||
if self.verbose:
|
||||
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
||||
return problems
|
||||
|
||||
if self.verbose:
|
||||
print(f" {jsonl_path} not found — using built-in 6-problem subset")
|
||||
return list(_BUILTIN_PROBLEMS)
|
||||
|
||||
def build_prompt(self, problem: dict[str, Any]) -> str:
|
||||
return (
|
||||
f"Edit the code in the workspace to satisfy this requirement:\n\n"
|
||||
f"{problem['instruction']}\n\n"
|
||||
f"Read the existing file(s), make the minimal changes needed, "
|
||||
f"and save the updated file(s)."
|
||||
)
|
||||
|
||||
def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
|
||||
setup = problem.get("setup_code", "")
|
||||
if setup:
|
||||
self._run_shell(setup, cwd=workspace, timeout=30.0)
|
||||
|
||||
# Write test file
|
||||
test_code = problem.get("test_code", "")
|
||||
if test_code:
|
||||
with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
|
||||
fh.write(test_code)
|
||||
|
||||
def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
|
||||
pid = problem.get("id", "unknown")
|
||||
code, output = self._run_shell(
|
||||
f"{sys.executable} test_harness.py", cwd=workspace, timeout=30.0
|
||||
)
|
||||
passed = code == 0 and "ALL_TESTS_PASSED" in output
|
||||
return BenchmarkResult(
|
||||
problem_id=pid, passed=passed, actual=output[:500],
|
||||
error="" if passed else output[:500],
|
||||
)
|
||||
Reference in New Issue
Block a user