Add 10 standard evaluation benchmark suites with CLI runner and README
Implements HumanEval, MBPP, SWE-Bench, Aider, LiveCodeBench (coding), MATH, GSM8K, AIME (math), and IFEval, BFCL (instruction following). Each suite includes built-in problem subsets (108 total) and supports loading full datasets from JSONL files. Includes comprehensive README with all commands. Agent-Logs-Url: https://github.com/HarnessLab/claw-code-agent/sessions/6890e3d0-3058-4b1f-b7e5-27171c079c62 Co-authored-by: abdoelsayed2016 <27821589+abdoelsayed2016@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
committed by
GitHub
GitHub
parent
3e32154618
commit
231b977b92
@@ -0,0 +1,232 @@
|
||||
"""
|
||||
IFEval benchmark suite.
|
||||
|
||||
IFEval (Instruction Following Evaluation) tests whether a model can
|
||||
follow verifiable natural-language instructions — e.g., "write at least
|
||||
200 words", "include exactly 3 bullet points", "respond in all lowercase".
|
||||
|
||||
Paper: https://arxiv.org/abs/2311.07911
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .base import BenchmarkResult, BenchmarkSuite
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in mini dataset — verifiable instruction-following tasks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BUILTIN_PROBLEMS: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "ifeval-001",
|
||||
"instruction": "Write a short paragraph about the benefits of exercise. Your response must be entirely in lowercase. Save it to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "all_lowercase", "file": "output.txt"},
|
||||
{"type": "min_words", "file": "output.txt", "value": 20},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-002",
|
||||
"instruction": "Write a poem about the ocean. It must have exactly 4 lines. Save it to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "exact_line_count", "file": "output.txt", "value": 4},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-003",
|
||||
"instruction": "List 5 programming languages. Format each as a numbered item (1. Language). Save to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "min_lines", "file": "output.txt", "value": 5},
|
||||
{"type": "contains_pattern", "file": "output.txt", "pattern": r"^\d+\.", "min_count": 5},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-004",
|
||||
"instruction": "Write a summary of what Python is in EXACTLY 3 sentences. Each sentence must end with a period. Save to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "exact_sentence_count", "file": "output.txt", "value": 3},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-005",
|
||||
"instruction": "Write a response that contains the word 'innovation' at least 3 times. The response should be about technology. Save to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "word_frequency", "file": "output.txt", "word": "innovation", "min_count": 3},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-006",
|
||||
"instruction": "Write a short paragraph about space exploration. It must contain exactly 2 bullet points, each starting with '- '. Save to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "contains_pattern", "file": "output.txt", "pattern": r"^- ", "min_count": 2, "max_count": 2},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-007",
|
||||
"instruction": "Write the numbers from 1 to 10, each on a separate line. Save to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "exact_line_count", "file": "output.txt", "value": 10},
|
||||
{"type": "contains", "file": "output.txt", "text": "1"},
|
||||
{"type": "contains", "file": "output.txt", "text": "10"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-008",
|
||||
"instruction": "Write a haiku (3 lines: 5 syllables, 7 syllables, 5 syllables) about nature. Save to output.txt. The response must be exactly 3 lines.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "exact_line_count", "file": "output.txt", "value": 3},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-009",
|
||||
"instruction": "Write a paragraph about artificial intelligence. Every sentence must start with the word 'AI'. Save to output.txt.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "sentences_start_with", "file": "output.txt", "prefix": "AI"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "ifeval-010",
|
||||
"instruction": "Create a JSON file called output.txt containing an object with exactly 3 keys: 'name', 'age', and 'city'. Values can be anything.",
|
||||
"checks": [
|
||||
{"type": "file_exists", "file": "output.txt"},
|
||||
{"type": "valid_json", "file": "output.txt"},
|
||||
{"type": "json_has_keys", "file": "output.txt", "keys": ["name", "age", "city"]},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _run_check(check: dict[str, Any], workspace: str) -> bool:
|
||||
"""Run a single verification check. Returns True if passed."""
|
||||
check_type = check["type"]
|
||||
filepath = os.path.join(workspace, check.get("file", "output.txt"))
|
||||
|
||||
if check_type == "file_exists":
|
||||
return os.path.exists(filepath)
|
||||
|
||||
if not os.path.exists(filepath):
|
||||
return False
|
||||
|
||||
with open(filepath) as fh:
|
||||
content = fh.read()
|
||||
|
||||
lines = [l for l in content.strip().split("\n") if l.strip()]
|
||||
|
||||
if check_type == "all_lowercase":
|
||||
# Check that alphabetic characters are lowercase
|
||||
alpha_chars = [c for c in content if c.isalpha()]
|
||||
return all(c.islower() for c in alpha_chars) if alpha_chars else True
|
||||
|
||||
if check_type == "min_words":
|
||||
words = content.split()
|
||||
return len(words) >= check["value"]
|
||||
|
||||
if check_type == "exact_line_count":
|
||||
return len(lines) == check["value"]
|
||||
|
||||
if check_type == "min_lines":
|
||||
return len(lines) >= check["value"]
|
||||
|
||||
if check_type == "contains_pattern":
|
||||
pattern = check["pattern"]
|
||||
matches = sum(1 for line in lines if re.search(pattern, line))
|
||||
if "min_count" in check and matches < check["min_count"]:
|
||||
return False
|
||||
if "max_count" in check and matches > check["max_count"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
if check_type == "exact_sentence_count":
|
||||
# Count sentences ending with . ! or ?
|
||||
sentences = re.split(r"[.!?]+", content.strip())
|
||||
sentences = [s.strip() for s in sentences if s.strip()]
|
||||
return len(sentences) == check["value"]
|
||||
|
||||
if check_type == "word_frequency":
|
||||
word = check["word"].lower()
|
||||
count = content.lower().count(word)
|
||||
return count >= check["min_count"]
|
||||
|
||||
if check_type == "contains":
|
||||
return check["text"] in content
|
||||
|
||||
if check_type == "sentences_start_with":
|
||||
prefix = check["prefix"]
|
||||
sentences = re.split(r"[.!?]\s+", content.strip())
|
||||
sentences = [s.strip() for s in sentences if s.strip()]
|
||||
return all(s.startswith(prefix) for s in sentences) if sentences else False
|
||||
|
||||
if check_type == "valid_json":
|
||||
try:
|
||||
json.loads(content)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
|
||||
if check_type == "json_has_keys":
|
||||
try:
|
||||
data = json.loads(content)
|
||||
return all(k in data for k in check["keys"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class IFEvalBenchmark(BenchmarkSuite):
|
||||
"""IFEval: instruction following evaluation."""
|
||||
|
||||
name = "IFEval"
|
||||
description = "Verifiable instruction-following evaluation"
|
||||
category = "instruction-following"
|
||||
|
||||
def load_dataset(self) -> list[dict[str, Any]]:
|
||||
jsonl_path = Path(self.data_dir) / "ifeval.jsonl"
|
||||
if jsonl_path.exists():
|
||||
problems: list[dict[str, Any]] = []
|
||||
with open(jsonl_path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line:
|
||||
problems.append(json.loads(line))
|
||||
if self.verbose:
|
||||
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
||||
return problems
|
||||
|
||||
if self.verbose:
|
||||
print(f" {jsonl_path} not found — using built-in 10-problem subset")
|
||||
return list(_BUILTIN_PROBLEMS)
|
||||
|
||||
def build_prompt(self, problem: dict[str, Any]) -> str:
|
||||
return problem["instruction"]
|
||||
|
||||
def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult:
|
||||
pid = problem.get("id", "unknown")
|
||||
checks = problem.get("checks", [])
|
||||
|
||||
failed_checks: list[str] = []
|
||||
for check in checks:
|
||||
if not _run_check(check, workspace):
|
||||
failed_checks.append(check["type"])
|
||||
|
||||
passed = len(failed_checks) == 0
|
||||
return BenchmarkResult(
|
||||
problem_id=pid, passed=passed,
|
||||
error=f"Failed checks: {', '.join(failed_checks)}" if failed_checks else "",
|
||||
metadata={"total_checks": len(checks), "failed_checks": failed_checks},
|
||||
)
|
||||
Reference in New Issue
Block a user