Add 7 new benchmark suites for Gemma 4 comparison
New suites: - MMLU-Pro: Professional-level 10-choice QA (14 subjects) - GPQA-Diamond: Graduate-level science QA - BigBench Extra Hard: Challenging reasoning tasks - MMMLU: Multilingual MMLU across 10 languages - HLE: Humanity's Last Exam (extremely hard) - Tau2: Tool-augmented reasoning (retail/airline/finance) - Codeforces: Competitive programming with ELO scoring Updates: - AIME now supports aime_2026.jsonl for 2026 problems - Registry expanded to 17 suites in 5 categories - download_datasets.py supports HF downloads for new suites - README.md updated with full documentation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
+14
-12
@@ -84,20 +84,22 @@ class AIMEBenchmark(BenchmarkSuite):
|
||||
category = "math"
|
||||
|
||||
def load_dataset(self) -> list[dict[str, Any]]:
|
||||
jsonl_path = Path(self.data_dir) / "aime.jsonl"
|
||||
if jsonl_path.exists():
|
||||
problems: list[dict[str, Any]] = []
|
||||
with open(jsonl_path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line:
|
||||
problems.append(json.loads(line))
|
||||
if self.verbose:
|
||||
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
||||
return problems
|
||||
# Prefer aime_2026.jsonl for AIME 2026 specific problems
|
||||
for fname in ("aime_2026.jsonl", "aime.jsonl"):
|
||||
jsonl_path = Path(self.data_dir) / fname
|
||||
if jsonl_path.exists():
|
||||
problems: list[dict[str, Any]] = []
|
||||
with open(jsonl_path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line:
|
||||
problems.append(json.loads(line))
|
||||
if self.verbose:
|
||||
print(f" Loaded {len(problems)} problems from {jsonl_path}")
|
||||
return problems
|
||||
|
||||
if self.verbose:
|
||||
print(f" {jsonl_path} not found — using built-in 10-problem subset")
|
||||
print(" No AIME data file found — using built-in 10-problem subset")
|
||||
return list(_BUILTIN_PROBLEMS)
|
||||
|
||||
def build_prompt(self, problem: dict[str, Any]) -> str:
|
||||
|
||||
Reference in New Issue
Block a user