""" IFEval benchmark suite. IFEval (Instruction Following Evaluation) tests whether a model can follow verifiable natural-language instructions — e.g., "write at least 200 words", "include exactly 3 bullet points", "respond in all lowercase". Paper: https://arxiv.org/abs/2311.07911 """ from __future__ import annotations import json import os import re from pathlib import Path from typing import Any from .base import BenchmarkResult, BenchmarkSuite # --------------------------------------------------------------------------- # Built-in mini dataset — verifiable instruction-following tasks # --------------------------------------------------------------------------- _BUILTIN_PROBLEMS: list[dict[str, Any]] = [ { "id": "ifeval-001", "instruction": "Write a short paragraph about the benefits of exercise. Your response must be entirely in lowercase. Save it to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "all_lowercase", "file": "output.txt"}, {"type": "min_words", "file": "output.txt", "value": 20}, ], }, { "id": "ifeval-002", "instruction": "Write a poem about the ocean. It must have exactly 4 lines. Save it to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "exact_line_count", "file": "output.txt", "value": 4}, ], }, { "id": "ifeval-003", "instruction": "List 5 programming languages. Format each as a numbered item (1. Language). Save to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "min_lines", "file": "output.txt", "value": 5}, {"type": "contains_pattern", "file": "output.txt", "pattern": r"^\d+\.", "min_count": 5}, ], }, { "id": "ifeval-004", "instruction": "Write a summary of what Python is in EXACTLY 3 sentences. Each sentence must end with a period. Save to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "exact_sentence_count", "file": "output.txt", "value": 3}, ], }, { "id": "ifeval-005", "instruction": "Write a response that contains the word 'innovation' at least 3 times. The response should be about technology. Save to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "word_frequency", "file": "output.txt", "word": "innovation", "min_count": 3}, ], }, { "id": "ifeval-006", "instruction": "Write a short paragraph about space exploration. It must contain exactly 2 bullet points, each starting with '- '. Save to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "contains_pattern", "file": "output.txt", "pattern": r"^- ", "min_count": 2, "max_count": 2}, ], }, { "id": "ifeval-007", "instruction": "Write the numbers from 1 to 10, each on a separate line. Save to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "exact_line_count", "file": "output.txt", "value": 10}, {"type": "contains", "file": "output.txt", "text": "1"}, {"type": "contains", "file": "output.txt", "text": "10"}, ], }, { "id": "ifeval-008", "instruction": "Write a haiku (3 lines: 5 syllables, 7 syllables, 5 syllables) about nature. Save to output.txt. The response must be exactly 3 lines.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "exact_line_count", "file": "output.txt", "value": 3}, ], }, { "id": "ifeval-009", "instruction": "Write a paragraph about artificial intelligence. Every sentence must start with the word 'AI'. Save to output.txt.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "sentences_start_with", "file": "output.txt", "prefix": "AI"}, ], }, { "id": "ifeval-010", "instruction": "Create a JSON file called output.txt containing an object with exactly 3 keys: 'name', 'age', and 'city'. Values can be anything.", "checks": [ {"type": "file_exists", "file": "output.txt"}, {"type": "valid_json", "file": "output.txt"}, {"type": "json_has_keys", "file": "output.txt", "keys": ["name", "age", "city"]}, ], }, ] def _run_check(check: dict[str, Any], workspace: str) -> bool: """Run a single verification check. Returns True if passed.""" check_type = check["type"] filepath = os.path.join(workspace, check.get("file", "output.txt")) if check_type == "file_exists": return os.path.exists(filepath) if not os.path.exists(filepath): return False with open(filepath) as fh: content = fh.read() lines = [l for l in content.strip().split("\n") if l.strip()] if check_type == "all_lowercase": # Check that alphabetic characters are lowercase alpha_chars = [c for c in content if c.isalpha()] return all(c.islower() for c in alpha_chars) if alpha_chars else True if check_type == "min_words": words = content.split() return len(words) >= check["value"] if check_type == "exact_line_count": return len(lines) == check["value"] if check_type == "min_lines": return len(lines) >= check["value"] if check_type == "contains_pattern": pattern = check["pattern"] matches = sum(1 for line in lines if re.search(pattern, line)) if "min_count" in check and matches < check["min_count"]: return False if "max_count" in check and matches > check["max_count"]: return False return True if check_type == "exact_sentence_count": # Count sentences ending with . ! or ? sentences = re.split(r"[.!?]+", content.strip()) sentences = [s.strip() for s in sentences if s.strip()] return len(sentences) == check["value"] if check_type == "word_frequency": word = check["word"].lower() count = content.lower().count(word) return count >= check["min_count"] if check_type == "contains": return check["text"] in content if check_type == "sentences_start_with": prefix = check["prefix"] sentences = re.split(r"[.!?]\s+", content.strip()) sentences = [s.strip() for s in sentences if s.strip()] return all(s.startswith(prefix) for s in sentences) if sentences else False if check_type == "valid_json": try: json.loads(content) return True except json.JSONDecodeError: return False if check_type == "json_has_keys": try: data = json.loads(content) return all(k in data for k in check["keys"]) except (json.JSONDecodeError, TypeError): return False return False class IFEvalBenchmark(BenchmarkSuite): """IFEval: instruction following evaluation.""" name = "IFEval" description = "Verifiable instruction-following evaluation" category = "instruction-following" def load_dataset(self) -> list[dict[str, Any]]: jsonl_path = Path(self.data_dir) / "ifeval.jsonl" if jsonl_path.exists(): problems: list[dict[str, Any]] = [] with open(jsonl_path) as fh: for line in fh: line = line.strip() if line: problems.append(json.loads(line)) if self.verbose: print(f" Loaded {len(problems)} problems from {jsonl_path}") return problems if self.verbose: print(f" {jsonl_path} not found — using built-in 10-problem subset") return list(_BUILTIN_PROBLEMS) def build_prompt(self, problem: dict[str, Any]) -> str: return problem["instruction"] def evaluate(self, problem: dict[str, Any], workspace: str) -> BenchmarkResult: pid = problem.get("id", "unknown") checks = problem.get("checks", []) failed_checks: list[str] = [] for check in checks: if not _run_check(check, workspace): failed_checks.append(check["type"]) passed = len(failed_checks) == 0 return BenchmarkResult( problem_id=pid, passed=passed, error=f"Failed checks: {', '.join(failed_checks)}" if failed_checks else "", metadata={"total_checks": len(checks), "failed_checks": failed_checks}, )