Add 7 new benchmark suites for Gemma 4 comparison

New suites:
- MMLU-Pro: Professional-level 10-choice QA (14 subjects)
- GPQA-Diamond: Graduate-level science QA
- BigBench Extra Hard: Challenging reasoning tasks
- MMMLU: Multilingual MMLU across 10 languages
- HLE: Humanity's Last Exam (extremely hard)
- Tau2: Tool-augmented reasoning (retail/airline/finance)
- Codeforces: Competitive programming with ELO scoring

Updates:
- AIME now supports aime_2026.jsonl for 2026 problems
- Registry expanded to 17 suites in 5 categories
- download_datasets.py supports HF downloads for new suites
- README.md updated with full documentation

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Abdelrahman Abdallah
2026-04-07 04:55:31 +02:00
parent c17c2768eb
commit 90489e7bfc
11 changed files with 1670 additions and 48 deletions
+14 -12
View File
@@ -84,20 +84,22 @@ class AIMEBenchmark(BenchmarkSuite):
category = "math"
def load_dataset(self) -> list[dict[str, Any]]:
jsonl_path = Path(self.data_dir) / "aime.jsonl"
if jsonl_path.exists():
problems: list[dict[str, Any]] = []
with open(jsonl_path) as fh:
for line in fh:
line = line.strip()
if line:
problems.append(json.loads(line))
if self.verbose:
print(f" Loaded {len(problems)} problems from {jsonl_path}")
return problems
# Prefer aime_2026.jsonl for AIME 2026 specific problems
for fname in ("aime_2026.jsonl", "aime.jsonl"):
jsonl_path = Path(self.data_dir) / fname
if jsonl_path.exists():
problems: list[dict[str, Any]] = []
with open(jsonl_path) as fh:
for line in fh:
line = line.strip()
if line:
problems.append(json.loads(line))
if self.verbose:
print(f" Loaded {len(problems)} problems from {jsonl_path}")
return problems
if self.verbose:
print(f" {jsonl_path} not found — using built-in 10-problem subset")
print(" No AIME data file found — using built-in 10-problem subset")
return list(_BUILTIN_PROBLEMS)
def build_prompt(self, problem: dict[str, Any]) -> str: