copilot-swe-agent[bot]
231b977b92
Implements HumanEval, MBPP, SWE-Bench, Aider, LiveCodeBench (coding), MATH, GSM8K, AIME (math), and IFEval, BFCL (instruction following). Each suite includes built-in problem subsets (108 total) and supports loading full datasets from JSONL files. Includes comprehensive README with all commands. Agent-Logs-Url: https://github.com/HarnessLab/claw-code-agent/sessions/6890e3d0-3058-4b1f-b7e5-27171c079c62 Co-authored-by: abdoelsayed2016 <27821589+abdoelsayed2016@users.noreply.github.com>
247 lines
22 KiB
Python
247 lines
22 KiB
Python
"""
|
|
HumanEval benchmark suite.
|
|
|
|
OpenAI's HumanEval evaluates code generation from docstrings.
|
|
164 hand-written Python programming problems with function signatures and
|
|
docstrings. The agent must produce a correct implementation.
|
|
|
|
Dataset: https://github.com/openai/human-eval
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .base import BenchmarkResult, BenchmarkSuite
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Built-in mini dataset (subset of 20 representative problems)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_BUILTIN_PROBLEMS: list[dict[str, Any]] = [
|
|
{
|
|
"task_id": "HumanEval/0",
|
|
"prompt": "from typing import List\n\n\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n given threshold.\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\n False\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\n True\n \"\"\"\n",
|
|
"canonical_solution": " for idx, elem in enumerate(numbers):\n for idx2, elem2 in enumerate(numbers):\n if idx != idx2:\n distance = abs(elem - elem2)\n if distance < threshold:\n return True\n\n return False\n",
|
|
"test": "def check(candidate):\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\n assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\n",
|
|
"entry_point": "has_close_elements",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/1",
|
|
"prompt": "from typing import List\n\n\ndef separate_paren_groups(paren_string: str) -> List[str]:\n \"\"\" Input to this function is a string containing multiple groups of nested parentheses. Your goal is to\n separate those group into separate strings and return the list of those.\n Separate groups are balanced (each open brace is properly closed) and not nested within each other\n Ignore any spaces in the input string.\n >>> separate_paren_groups('( ) (( )) (( )( ))')\n ['()', '(())', '(()())']\n \"\"\"\n",
|
|
"canonical_solution": " result = []\n current_string = []\n current_depth = 0\n\n for c in paren_string:\n if c == '(':\n current_depth += 1\n current_string.append(c)\n elif c == ')':\n current_depth -= 1\n current_string.append(c)\n\n if current_depth == 0:\n result.append(''.join(current_string))\n current_string.clear()\n\n return result\n",
|
|
"test": "def check(candidate):\n assert candidate('(()()) ((())) () ((())()())') == ['(()())', '((()))', '()', '((())()())']\n assert candidate('() (()) ((())) (((())))') == ['()', '(())', '((()))', '(((())))']\n assert candidate('(()(()))') == ['(()(()))']\n assert candidate('( ) (( )) (( )( ))') == ['()', '(())', '(()())']\n",
|
|
"entry_point": "separate_paren_groups",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/2",
|
|
"prompt": "\n\ndef truncate_number(number: float) -> float:\n \"\"\" Given a positive floating point number, it can be decomposed into\n and integer part (largest integer smaller than given number) and decimals\n (leftover part always smaller than 1).\n\n Return the decimal part of the number.\n >>> truncate_number(3.5)\n 0.5\n \"\"\"\n",
|
|
"canonical_solution": " return number % 1.0\n",
|
|
"test": "def check(candidate):\n assert candidate(3.5) == 0.5\n assert abs(candidate(1.33) - 0.33) < 1e-6\n assert abs(candidate(123.456) - 0.456) < 1e-6\n",
|
|
"entry_point": "truncate_number",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/3",
|
|
"prompt": "from typing import List\n\n\ndef below_zero(operations: List[int]) -> bool:\n \"\"\" You're given a list of deposit and withdrawal operations on a bank account that starts with\n zero balance. Your task is to detect if at any point the balance of account falls below zero, and\n at that point function should return True. Otherwise it should return False.\n >>> below_zero([1, 2, 3])\n False\n >>> below_zero([1, 2, -4, 5])\n True\n \"\"\"\n",
|
|
"canonical_solution": " balance = 0\n\n for op in operations:\n balance += op\n if balance < 0:\n return True\n\n return False\n",
|
|
"test": "def check(candidate):\n assert candidate([]) == False\n assert candidate([1, 2, -3, 1, 2, -3]) == False\n assert candidate([1, 2, -4, 5, 6]) == True\n assert candidate([1, -1, 2, -2, 5, -5, 4, -4]) == False\n assert candidate([1, -1, 2, -2, 5, -5, 4, -5]) == True\n assert candidate([1, -2, 2, -2, 5, -5, 4, -4]) == True\n",
|
|
"entry_point": "below_zero",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/4",
|
|
"prompt": "from typing import List\n\n\ndef mean_absolute_deviation(numbers: List[float]) -> float:\n \"\"\" For a given list of input numbers, calculate Mean Absolute Deviation\n around the mean of this dataset.\n Mean Absolute Deviation is the average absolute difference between each\n element and a centerpoint (mean in this case):\n MAD = average | x - x_mean |\n >>> mean_absolute_deviation([1.0, 2.0, 3.0, 4.0])\n 1.0\n \"\"\"\n",
|
|
"canonical_solution": " mean = sum(numbers) / len(numbers)\n return sum(abs(x - mean) for x in numbers) / len(numbers)\n",
|
|
"test": "def check(candidate):\n assert abs(candidate([1.0, 2.0, 3.0]) - 2.0/3.0) < 1e-6\n assert abs(candidate([1.0, 2.0, 3.0, 4.0]) - 1.0) < 1e-6\n assert abs(candidate([1.0, 2.0, 3.0, 4.0, 5.0]) - 1.2) < 1e-6\n",
|
|
"entry_point": "mean_absolute_deviation",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/5",
|
|
"prompt": "from typing import List\n\n\ndef intersperse(numbers: List[int], delimeter: int) -> List[int]:\n \"\"\" Insert a number 'delimeter' between every two consecutive elements of input list `numbers'\n >>> intersperse([], 4)\n []\n >>> intersperse([1, 2, 3], 4)\n [1, 4, 2, 4, 3]\n \"\"\"\n",
|
|
"canonical_solution": " if not numbers:\n return []\n\n result = []\n\n for n in numbers[:-1]:\n result.append(n)\n result.append(delimeter)\n\n result.append(numbers[-1])\n\n return result\n",
|
|
"test": "def check(candidate):\n assert candidate([], 7) == []\n assert candidate([5, 6, 3, 2], 8) == [5, 8, 6, 8, 3, 8, 2]\n assert candidate([2, 2, 2], 2) == [2, 2, 2, 2, 2]\n",
|
|
"entry_point": "intersperse",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/6",
|
|
"prompt": "from typing import List\n\n\ndef parse_nested_parens(paren_string: str) -> List[int]:\n \"\"\" Input to this function is a string represented multiple groups of nested parentheses separated by spaces.\n For each of the group, output the deepest level of nesting of parentheses.\n E.g. (()()) has maximum two levels of nesting while ((())) has three.\n\n >>> parse_nested_parens('(()()) ((())) () ((())()())')\n [2, 3, 1, 3]\n \"\"\"\n",
|
|
"canonical_solution": " def parse_paren_group(s):\n depth = 0\n max_depth = 0\n for c in s:\n if c == '(':\n depth += 1\n max_depth = max(depth, max_depth)\n else:\n depth -= 1\n\n return max_depth\n\n return [parse_paren_group(x) for x in paren_string.split(' ') if x]\n",
|
|
"test": "def check(candidate):\n assert candidate('(()()) ((())) () ((())()())') == [2, 3, 1, 3]\n assert candidate('() (()) ((())) (((())))') == [1, 2, 3, 4]\n assert candidate('(()(())((())))') == [4]\n",
|
|
"entry_point": "parse_nested_parens",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/7",
|
|
"prompt": "from typing import List\n\n\ndef filter_by_substring(strings: List[str], substring: str) -> List[str]:\n \"\"\" Filter an input list of strings only for ones that contain given substring\n >>> filter_by_substring([], 'a')\n []\n >>> filter_by_substring(['abc', 'bacd', 'cde', 'array'], 'a')\n ['abc', 'bacd', 'array']\n \"\"\"\n",
|
|
"canonical_solution": " return [x for x in strings if substring in x]\n",
|
|
"test": "def check(candidate):\n assert candidate([], 'john') == []\n assert candidate(['xxx', 'asd', 'xxy', 'john doe', 'xxxuj', 'xxx'], 'xxx') == ['xxx', 'xxxuj', 'xxx']\n assert candidate(['xxx', 'asd', 'aaber', 'john doe', 'xxxuj', 'xxx'], 'john') == ['john doe']\n assert candidate(['grunt', 'hierarchies', 'aaaxyz'], 'hi') == ['hierarchies']\n",
|
|
"entry_point": "filter_by_substring",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/8",
|
|
"prompt": "from typing import List, Tuple\n\n\ndef sum_product(numbers: List[int]) -> Tuple[int, int]:\n \"\"\" For a given list of integers, return a tuple consisting of a sum and a product of all the integers in a list.\n Empty sum should be equal to 0 and empty product should be equal to 1.\n >>> sum_product([])\n (0, 1)\n >>> sum_product([1, 2, 3, 4])\n (10, 24)\n \"\"\"\n",
|
|
"canonical_solution": " sum_value = 0\n prod_value = 1\n\n for n in numbers:\n sum_value += n\n prod_value *= n\n\n return sum_value, prod_value\n",
|
|
"test": "def check(candidate):\n assert candidate([]) == (0, 1)\n assert candidate([1, 1, 1]) == (3, 1)\n assert candidate([100, 0]) == (100, 0)\n assert candidate([3, 5, 7]) == (15, 105)\n assert candidate([10]) == (10, 10)\n",
|
|
"entry_point": "sum_product",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/9",
|
|
"prompt": "from typing import List, Tuple\n\n\ndef rolling_max(numbers: List[int]) -> List[int]:\n \"\"\" From a given list of integers, generate a list of rolling maximum element found until given moment\n in the sequence.\n >>> rolling_max([1, 2, 3, 2, 3, 4, 2])\n [1, 2, 3, 3, 3, 4, 4]\n \"\"\"\n",
|
|
"canonical_solution": " running_max = None\n result = []\n\n for n in numbers:\n if running_max is None:\n running_max = n\n else:\n running_max = max(running_max, n)\n\n result.append(running_max)\n\n return result\n",
|
|
"test": "def check(candidate):\n assert candidate([]) == []\n assert candidate([1, 2, 3, 4]) == [1, 2, 3, 4]\n assert candidate([4, 3, 2, 1]) == [4, 4, 4, 4]\n assert candidate([3, 3, 3, 3]) == [3, 3, 3, 3]\n",
|
|
"entry_point": "rolling_max",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/10",
|
|
"prompt": "\n\ndef is_palindrome(string: str) -> bool:\n \"\"\" Test if given string is a palindrome \"\"\"\n return string == string[::-1]\n\n\ndef make_palindrome(string: str) -> str:\n \"\"\" Find the shortest palindrome that begins with a supplied string.\n Algorithm idea is simple:\n - Find the longest postfix of supplied string that is a palindrome.\n - Append to the end of the string reverse of a string prefix that comes before the palindromic suffix.\n >>> make_palindrome('')\n ''\n >>> make_palindrome('cat')\n 'catac'\n >>> make_palindrome('cata')\n 'catac'\n \"\"\"\n",
|
|
"canonical_solution": " if not string:\n return ''\n\n beginning_of_suffix = 0\n\n while not is_palindrome(string[beginning_of_suffix:]):\n beginning_of_suffix += 1\n\n return string + string[:beginning_of_suffix][::-1]\n",
|
|
"test": "def check(candidate):\n assert candidate('') == ''\n assert candidate('x') == 'x'\n assert candidate('xyz') == 'xyzyx'\n assert candidate('xyx') == 'xyx'\n assert candidate('jerry') == 'jerryrrej'\n",
|
|
"entry_point": "make_palindrome",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/11",
|
|
"prompt": "from typing import List\n\n\ndef string_xor(a: str, b: str) -> str:\n \"\"\" Input are two strings a and b consisting only of 1s and 0s.\n Perform binary XOR on these inputs and return result also as a string.\n >>> string_xor('010', '110')\n '100'\n \"\"\"\n",
|
|
"canonical_solution": " def xor(i, j):\n if i == j:\n return '0'\n else:\n return '1'\n\n return ''.join(xor(x, y) for x, y in zip(a, b))\n",
|
|
"test": "def check(candidate):\n assert candidate('111000', '101010') == '010010'\n assert candidate('1', '1') == '0'\n assert candidate('0101', '0000') == '0101'\n",
|
|
"entry_point": "string_xor",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/12",
|
|
"prompt": "from typing import List, Optional\n\n\ndef longest(strings: List[str]) -> Optional[str]:\n \"\"\" Out of list of strings, return the longest one. Return the first one in case of multiple\n strings of the same length. Return None in case the input list is empty.\n >>> longest([])\n\n >>> longest(['a', 'b', 'c'])\n 'a'\n >>> longest(['a', 'bb', 'ccc'])\n 'ccc'\n \"\"\"\n",
|
|
"canonical_solution": " if not strings:\n return None\n\n maxlen = max(len(x) for x in strings)\n for s in strings:\n if len(s) == maxlen:\n return s\n",
|
|
"test": "def check(candidate):\n assert candidate([]) == None\n assert candidate(['x', 'y', 'z']) == 'x'\n assert candidate(['x', 'yyy', 'zzzz', 'www', 'kkkk', 'abc']) == 'zzzz'\n",
|
|
"entry_point": "longest",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/13",
|
|
"prompt": "\n\ndef greatest_common_divisor(a: int, b: int) -> int:\n \"\"\" Return a greatest common divisor of two integers a and b\n >>> greatest_common_divisor(3, 5)\n 1\n >>> greatest_common_divisor(25, 15)\n 5\n \"\"\"\n",
|
|
"canonical_solution": " while b:\n a, b = b, a % b\n return a\n",
|
|
"test": "def check(candidate):\n assert candidate(3, 7) == 1\n assert candidate(10, 15) == 5\n assert candidate(49, 14) == 7\n assert candidate(144, 60) == 12\n",
|
|
"entry_point": "greatest_common_divisor",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/14",
|
|
"prompt": "from typing import List\n\n\ndef all_prefixes(string: str) -> List[str]:\n \"\"\" Return list of all prefixes from shortest to longest of the input string\n >>> all_prefixes('abc')\n ['a', 'ab', 'abc']\n \"\"\"\n",
|
|
"canonical_solution": " result = []\n\n for i in range(len(string)):\n result.append(string[:i+1])\n return result\n",
|
|
"test": "def check(candidate):\n assert candidate('') == []\n assert candidate('asdfgh') == ['a', 'as', 'asd', 'asdf', 'asdfe', 'asdfgh']\n assert candidate('WWW') == ['W', 'WW', 'WWW']\n",
|
|
"entry_point": "all_prefixes",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/15",
|
|
"prompt": "\n\ndef string_sequence(n: int) -> str:\n \"\"\" Return a string containing space-delimited numbers starting from 0 upto n inclusive.\n >>> string_sequence(0)\n '0'\n >>> string_sequence(5)\n '0 1 2 3 4 5'\n \"\"\"\n",
|
|
"canonical_solution": " return ' '.join([str(x) for x in range(n + 1)])\n",
|
|
"test": "def check(candidate):\n assert candidate(0) == '0'\n assert candidate(3) == '0 1 2 3'\n assert candidate(10) == '0 1 2 3 4 5 6 7 8 9 10'\n",
|
|
"entry_point": "string_sequence",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/16",
|
|
"prompt": "\n\ndef count_distinct_characters(string: str) -> int:\n \"\"\" Given a string, find out how many distinct characters (regardless of case) does it consist of\n >>> count_distinct_characters('xyzXYZ')\n 3\n >>> count_distinct_characters('Jerry')\n 4\n \"\"\"\n",
|
|
"canonical_solution": " return len(set(string.lower()))\n",
|
|
"test": "def check(candidate):\n assert candidate('') == 0\n assert candidate('abcde') == 5\n assert candidate('abcde' + 'cade' + 'CADE') == 5\n assert candidate('aaaaAAAAaaaa') == 1\n assert candidate('Jerry jbread') == 7\n",
|
|
"entry_point": "count_distinct_characters",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/17",
|
|
"prompt": "from typing import List\n\n\ndef parse_music(music_string: str) -> List[int]:\n \"\"\" Input to this function is a string representing musical notes in a special ASCII format.\n Your task is to parse this string and return list of integers corresponding to how many beats does each\n not last.\n\n Here is a legend:\n 'o' - whole note, lasts four beats\n 'o|' - half note, lasts two beats\n '.|' - quarter note, lasts one beat\n\n >>> parse_music('o o| .| o| o| .| .| .| .| o o')\n [4, 2, 1, 2, 2, 1, 1, 1, 1, 4, 4]\n \"\"\"\n",
|
|
"canonical_solution": " note_map = {'o': 4, 'o|': 2, '.|': 1}\n return [note_map[x] for x in music_string.split(' ') if x]\n",
|
|
"test": "def check(candidate):\n assert candidate('') == []\n assert candidate('o o o o') == [4, 4, 4, 4]\n assert candidate('.| .| .| .|') == [1, 1, 1, 1]\n assert candidate('o| o| .| .| o o o o') == [2, 2, 1, 1, 4, 4, 4, 4]\n assert candidate('o| .| o| .| o o| o o|') == [2, 1, 2, 1, 4, 2, 4, 2]\n",
|
|
"entry_point": "parse_music",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/18",
|
|
"prompt": "\n\ndef how_many_times(string: str, substring: str) -> int:\n \"\"\" Find how many times a given substring can be found in the original string. Count overlapping cases.\n >>> how_many_times('', 'a')\n 0\n >>> how_many_times('aaa', 'a')\n 3\n >>> how_many_times('aaaa', 'aa')\n 3\n \"\"\"\n",
|
|
"canonical_solution": " times = 0\n\n for i in range(len(string) - len(substring) + 1):\n if string[i:i+len(substring)] == substring:\n times += 1\n\n return times\n",
|
|
"test": "def check(candidate):\n assert candidate('', 'x') == 0\n assert candidate('xyxyxyx', 'x') == 4\n assert candidate('cacacacac', 'cac') == 4\n assert candidate('john doe', 'john') == 1\n",
|
|
"entry_point": "how_many_times",
|
|
},
|
|
{
|
|
"task_id": "HumanEval/19",
|
|
"prompt": "from typing import List\n\n\ndef sort_numbers(numbers: str) -> str:\n \"\"\" Input is a space-delimited string of numberals from 'zero' to 'nine'.\n Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'.\n Return the string with numbers sorted from smallest to largest\n >>> sort_numbers('three one five')\n 'one three five'\n \"\"\"\n",
|
|
"canonical_solution": " value_map = {\n 'zero': 0,\n 'one': 1,\n 'two': 2,\n 'three': 3,\n 'four': 4,\n 'five': 5,\n 'six': 6,\n 'seven': 7,\n 'eight': 8,\n 'nine': 9\n }\n return ' '.join(sorted([x for x in numbers.split(' ') if x], key=lambda x: value_map[x]))\n",
|
|
"test": "def check(candidate):\n assert candidate('') == ''\n assert candidate('three') == 'three'\n assert candidate('three five nine') == 'three five nine'\n assert candidate('five zero four seven nine eight') == 'zero four five seven eight nine'\n assert candidate('six five four three two one zero') == 'zero one two three four five six'\n",
|
|
"entry_point": "sort_numbers",
|
|
},
|
|
]
|
|
|
|
|
|
class HumanEvalBenchmark(BenchmarkSuite):
|
|
"""HumanEval: code generation from docstrings (164 problems)."""
|
|
|
|
name = "HumanEval"
|
|
description = "Code generation from Python function docstrings"
|
|
category = "coding"
|
|
|
|
def load_dataset(self) -> list[dict[str, Any]]:
|
|
"""Load HumanEval problems.
|
|
|
|
Checks for a JSONL file at ``data_dir/humaneval.jsonl`` first.
|
|
Falls back to the built-in 20-problem subset.
|
|
"""
|
|
jsonl_path = Path(self.data_dir) / "humaneval.jsonl"
|
|
if jsonl_path.exists():
|
|
problems: list[dict[str, Any]] = []
|
|
with open(jsonl_path) as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if line:
|
|
problems.append(json.loads(line))
|
|
if self.verbose:
|
|
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
|
return problems
|
|
|
|
if self.verbose:
|
|
print(
|
|
f" {jsonl_path} not found — using built-in 20-problem subset"
|
|
)
|
|
return list(_BUILTIN_PROBLEMS)
|
|
|
|
def build_prompt(self, problem: dict[str, Any]) -> str:
|
|
prompt_code = problem["prompt"]
|
|
entry = problem["entry_point"]
|
|
return (
|
|
f"Complete the following Python function. Write ONLY the function body. "
|
|
f"Save the complete file (including the signature shown below) as solution.py.\n\n"
|
|
f"```python\n{prompt_code}```\n\n"
|
|
f"The function name is `{entry}`. Make sure the file is valid Python."
|
|
)
|
|
|
|
def setup_workspace(self, problem: dict[str, Any], workspace: str) -> None:
|
|
# Write the test harness so we can verify after the agent runs
|
|
test_code = problem["test"]
|
|
entry = problem["entry_point"]
|
|
harness = textwrap.dedent(f"""\
|
|
import sys
|
|
sys.path.insert(0, ".")
|
|
from solution import {entry}
|
|
|
|
{test_code}
|
|
|
|
check({entry})
|
|
print("ALL_TESTS_PASSED")
|
|
""")
|
|
with open(os.path.join(workspace, "test_harness.py"), "w") as fh:
|
|
fh.write(harness)
|
|
|
|
def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
|
|
pid = problem.get("task_id", problem.get("id", "unknown"))
|
|
sol = os.path.join(workspace, "solution.py")
|
|
if not os.path.exists(sol):
|
|
return BenchmarkResult(
|
|
problem_id=pid, passed=False, error="solution.py not found"
|
|
)
|
|
|
|
code, output = self._run_shell(
|
|
f"{sys.executable} test_harness.py", cwd=workspace, timeout=30.0
|
|
)
|
|
passed = code == 0 and "ALL_TESTS_PASSED" in output
|
|
return BenchmarkResult(
|
|
problem_id=pid,
|
|
passed=passed,
|
|
actual=output[:500],
|
|
error="" if passed else output[:500],
|
|
)
|