{ "schema_version": 1, "protocol_id": "llm-atlas-deepseek-chat-cross-source-sampling-v1", "inputs": { "sampling": { "path": "/tmp/deepseek-v2-lite-chat-cross-source-sampling-formal.json", "sha256": "f013132485f27adce008f03f781bed9982efc0d7939f13faede01c9f6f3d7f7c", "content_hash": "8fa14db0e9e6f1fd649a500796953ca948606a2c525d9c92c4b5f848beeaa3a7" }, "evaluation": { "path": "/tmp/deepseek-v2-lite-chat-cross-source-sampling-eval.json", "sha256": "e88b274599fc5951561f9e5e7438fb4d6d25f341ae3bc4a2c4121877a0db8975", "content_hash": "66326e89a92e4c3b477bd813ca9b7139a77312341cdd6b1ccc32fe557380d021" }, "reproduction": { "path": "/tmp/deepseek-v2-lite-chat-cross-source-sampling-reproduction.json", "sha256": "ec4a47894953f5d73bb62211b588632ef7032da0e915218992b7fc3ef9c2a556", "summary": { "cells": 64, "all_preregistered_fields_exact": 64, "by_field": { "run_seed_exact": 64, "prompt_hash_exact": 64, "generated_token_ids_exact": 64, "decoded_text_exact": 64, "eos_state_exact": 64, "truncation_state_exact": 64, "cpu_rng_pre_state_exact": 64, "cuda_rng_pre_state_exact": 64 } } } }, "contract": { "source_is_primary_coverage_unit": true, "seed_is_within_source_repeat": true, "sources": 16, "sources_per_domain": 4, "seeds_per_source_condition": 4, "conditions": [ "s0_eos", "s1_eos", "s0_period", "s1_period" ], "outputs": 256, "source_order": [ "wikitext2/raw-validation/0443", "wikitext2/raw-validation/0030", "wikitext2/raw-validation/2909", "wikitext2/raw-validation/2746", "tnews/test/4855", "tnews/test/8935", "tnews/test/3448", "tnews/test/1059", "HumanEval/31", "HumanEval/44", "HumanEval/133", "HumanEval/23", "gsm8k/test/1069", "gsm8k/test/1228", "gsm8k/test/0144", "gsm8k/test/1251" ], "sources_by_domain": { "english": [ "wikitext2/raw-validation/0443", "wikitext2/raw-validation/0030", "wikitext2/raw-validation/2909", "wikitext2/raw-validation/2746" ], "chinese": [ "tnews/test/4855", "tnews/test/8935", "tnews/test/3448", "tnews/test/1059" ], "code": [ "HumanEval/31", "HumanEval/44", "HumanEval/133", "HumanEval/23" ], "math": [ "gsm8k/test/1069", "gsm8k/test/1228", "gsm8k/test/0144", "gsm8k/test/1251" ] }, "no_population_confidence_intervals": true, "no_p_values": true }, "overall": { "outputs": 256, "natural_eos": 250, "budget_truncated": 6, "unique_trajectory_hashes": 247, "math": { "outputs": 64, "fixed_budget_success": 47, "strict_complete_success": 47 }, "code": { "outputs": 64, "fixed_budget_success": 52, "strict_complete_success": 52 } }, "source_metadata": { "wikitext2/raw-validation/0443": { "domain": "english", "within_domain_index": 0, "selection_rank": "004b632f55ea14a7a1f99552db185df62bb0bbd0914ca94c411d0325d0e96639" }, "wikitext2/raw-validation/0030": { "domain": "english", "within_domain_index": 1, "selection_rank": "0064700bc7cd279cd9669445fce1a5a410d6e0cae091efed9c8f0182e47d7ab0" }, "wikitext2/raw-validation/2909": { "domain": "english", "within_domain_index": 2, "selection_rank": "00a0c873d028d892e454883e687d7af51aee2831fe2f7884668ca6dfe0d681c8" }, "wikitext2/raw-validation/2746": { "domain": "english", "within_domain_index": 3, "selection_rank": "00c7a9a1b242b365225d7fdeec777508a5a3d90a76a7cf4cd2e93cee8f1693f4" }, "tnews/test/4855": { "domain": "chinese", "within_domain_index": 0, "selection_rank": "000b4d139c925918017352dc899d44353839fb496e1a607335490e402857bf46" }, "tnews/test/8935": { "domain": "chinese", "within_domain_index": 1, "selection_rank": "000c9943109217e6b0473de9c090009fae3ee782aa8e3c63f08abe82cb8c487e" }, "tnews/test/3448": { "domain": "chinese", "within_domain_index": 2, "selection_rank": "001d2b482c8c5d11c08d0fe0d5ce8421759f1ae362e4d6ab339a98cbcf893e3d" }, "tnews/test/1059": { "domain": "chinese", "within_domain_index": 3, "selection_rank": "003c8ebfb96b25b85cd0d61695249921a80438fbed1d314631b98720f7e9d0bd" }, "HumanEval/31": { "domain": "code", "within_domain_index": 0, "selection_rank": "004f4cfa62936a4a096e53df7d0ef3926c3ee97d32cf67a3c9bd64569f580e5c" }, "HumanEval/44": { "domain": "code", "within_domain_index": 1, "selection_rank": "030cfc7acc71ac4484d25020b2f340973c6cd1e20c2bdb1b3be9d532032eaa24" }, "HumanEval/133": { "domain": "code", "within_domain_index": 2, "selection_rank": "03661faa70d0db8ba0c1d391356f9c7cdb8f50ccbeedc0911f6b2a3be0190ed3" }, "HumanEval/23": { "domain": "code", "within_domain_index": 3, "selection_rank": "03b3c7329e474bf8e9660969ed5898d561e718f12a59d2fc1ed641be54b68860" }, "gsm8k/test/1069": { "domain": "math", "within_domain_index": 0, "selection_rank": "000946b459ce33a645f403b9d781a7aefd81f5568412149f4588381fed55b1d7" }, "gsm8k/test/1228": { "domain": "math", "within_domain_index": 1, "selection_rank": "002201d4cbc0b85d338597dcb68c3ffd4d00ef12b7b44e352c5bc1911d6fa984" }, "gsm8k/test/0144": { "domain": "math", "within_domain_index": 2, "selection_rank": "004a245538bdc42cc175a178c73dfaaaebc9713e3215ffcdfb9400fb63b3a941" }, "gsm8k/test/1251": { "domain": "math", "within_domain_index": 3, "selection_rank": "0056e66a635febc5b2ace25221ed8fe25366682898fc2d4d0132b30812f59e30" } }, "source_condition_cells": { "wikitext2/raw-validation/0443": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 377, "median": 362.0, "min": 316, "max": 468 }, "generated_token_seed_range": 152, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 420.75, "median": 408.0, "min": 355, "max": 512 }, "generated_token_seed_range": 157, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 3, "natural_eos_rate": 0.75, "budget_truncated": 1, "generated_tokens": { "count": 4, "mean": 411, "median": 406.0, "min": 320, "max": 512 }, "generated_token_seed_range": 192, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 3, "BUDGET_TRUNCATED_UNRESOLVED": 1 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 144.75, "median": 134.5, "min": 121, "max": 189 }, "generated_token_seed_range": 68, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "wikitext2/raw-validation/0030": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 265, "median": 272.5, "min": 64, "max": 451 }, "generated_token_seed_range": 387, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 64, "median": 65.5, "min": 57, "max": 68 }, "generated_token_seed_range": 11, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 356.25, "median": 411.5, "min": 105, "max": 497 }, "generated_token_seed_range": 392, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 61.75, "median": 67.5, "min": 41, "max": 71 }, "generated_token_seed_range": 30, "unique_trajectory_hashes": 3, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "wikitext2/raw-validation/2909": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 220.5, "median": 218.0, "min": 147, "max": 299 }, "generated_token_seed_range": 152, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 189, "median": 194.5, "min": 108, "max": 259 }, "generated_token_seed_range": 151, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 277.5, "median": 279.0, "min": 249, "max": 303 }, "generated_token_seed_range": 54, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 214, "median": 228.0, "min": 115, "max": 285 }, "generated_token_seed_range": 170, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "wikitext2/raw-validation/2746": { "s0_eos": { "outputs": 4, "natural_eos": 3, "natural_eos_rate": 0.75, "budget_truncated": 1, "generated_tokens": { "count": 4, "mean": 404.75, "median": 415.0, "min": 277, "max": 512 }, "generated_token_seed_range": 235, "unique_trajectory_hashes": 4, "completion_classes": { "BUDGET_TRUNCATED_UNRESOLVED": 1, "NATURAL_EOS": 3 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 325.25, "median": 323.5, "min": 259, "max": 395 }, "generated_token_seed_range": 136, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 2, "natural_eos_rate": 0.5, "budget_truncated": 2, "generated_tokens": { "count": 4, "mean": 468, "median": 473.5, "min": 413, "max": 512 }, "generated_token_seed_range": 99, "unique_trajectory_hashes": 4, "completion_classes": { "BUDGET_TRUNCATED_UNRESOLVED": 2, "NATURAL_EOS": 2 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 264.75, "median": 256.0, "min": 246, "max": 301 }, "generated_token_seed_range": 55, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "tnews/test/4855": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 94.25, "median": 94.5, "min": 69, "max": 119 }, "generated_token_seed_range": 50, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 96, "median": 93.0, "min": 77, "max": 121 }, "generated_token_seed_range": 44, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 101.75, "median": 106.0, "min": 75, "max": 120 }, "generated_token_seed_range": 45, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 89.25, "median": 87.5, "min": 77, "max": 105 }, "generated_token_seed_range": 28, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "tnews/test/8935": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 224.25, "median": 247.5, "min": 129, "max": 273 }, "generated_token_seed_range": 144, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 262.75, "median": 254.0, "min": 90, "max": 453 }, "generated_token_seed_range": 363, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 200.75, "median": 208.5, "min": 98, "max": 288 }, "generated_token_seed_range": 190, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 226.25, "median": 218.0, "min": 207, "max": 262 }, "generated_token_seed_range": 55, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "tnews/test/3448": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 119.25, "median": 124.5, "min": 87, "max": 141 }, "generated_token_seed_range": 54, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 128.25, "median": 135.5, "min": 92, "max": 150 }, "generated_token_seed_range": 58, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 98, "median": 98.0, "min": 91, "max": 105 }, "generated_token_seed_range": 14, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 134.5, "median": 87.5, "min": 87, "max": 276 }, "generated_token_seed_range": 189, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "tnews/test/1059": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 193.5, "median": 196.0, "min": 149, "max": 233 }, "generated_token_seed_range": 84, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 193.25, "median": 216.5, "min": 95, "max": 245 }, "generated_token_seed_range": 150, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 251.75, "median": 244.5, "min": 235, "max": 283 }, "generated_token_seed_range": 48, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 98, "median": 100.5, "min": 75, "max": 116 }, "generated_token_seed_range": 41, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": null, "fixed_budget_success_rate": null, "strict_complete_success": null, "strict_complete_success_rate": null, "observed_any_fixed_budget_pass": null, "observed_any_strict_complete_pass": null, "seed_success_range": null } }, "HumanEval/31": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 451, "median": 467.5, "min": 379, "max": 490 }, "generated_token_seed_range": 111, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 385.25, "median": 379.0, "min": 350, "max": 433 }, "generated_token_seed_range": 83, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 420, "median": 419.5, "min": 382, "max": 459 }, "generated_token_seed_range": 77, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 59, "median": 59.0, "min": 59, "max": 59 }, "generated_token_seed_range": 0, "unique_trajectory_hashes": 1, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } } }, "HumanEval/44": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 474.5, "median": 476.5, "min": 437, "max": 508 }, "generated_token_seed_range": 71, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 2, "fixed_budget_success_rate": 0.5, "strict_complete_success": 2, "strict_complete_success_rate": 0.5, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "runtime_error": 1, "assertion_failed": 1, "passed": 2 } } }, "s1_eos": { "outputs": 4, "natural_eos": 3, "natural_eos_rate": 0.75, "budget_truncated": 1, "generated_tokens": { "count": 4, "mean": 433.5, "median": 425.0, "min": 372, "max": 512 }, "generated_token_seed_range": 140, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 3, "TASK_TERMINAL_BEFORE_EOS": 1 }, "fixed_budget_success": 1, "fixed_budget_success_rate": 0.25, "strict_complete_success": 1, "strict_complete_success_rate": 0.25, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "runtime_error": 3, "passed": 1 } } }, "s0_period": { "outputs": 4, "natural_eos": 3, "natural_eos_rate": 0.75, "budget_truncated": 1, "generated_tokens": { "count": 4, "mean": 432.5, "median": 476.0, "min": 266, "max": 512 }, "generated_token_seed_range": 246, "unique_trajectory_hashes": 4, "completion_classes": { "BUDGET_TRUNCATED_UNRESOLVED": 1, "NATURAL_EOS": 3 }, "fixed_budget_success": 1, "fixed_budget_success_rate": 0.25, "strict_complete_success": 1, "strict_complete_success_rate": 0.25, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "assertion_failed": 2, "runtime_error": 1, "passed": 1 } } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 146.75, "median": 164.0, "min": 79, "max": 180 }, "generated_token_seed_range": 101, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } } }, "HumanEval/133": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 386.75, "median": 388.5, "min": 366, "max": 404 }, "generated_token_seed_range": 38, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 362.75, "median": 360.0, "min": 355, "max": 376 }, "generated_token_seed_range": 21, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 343.25, "median": 341.0, "min": 329, "max": 362 }, "generated_token_seed_range": 33, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 70.5, "median": 76.0, "min": 52, "max": 78 }, "generated_token_seed_range": 26, "unique_trajectory_hashes": 3, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 0, "fixed_budget_success_rate": 0, "strict_complete_success": 0, "strict_complete_success_rate": 0, "observed_any_fixed_budget_pass": false, "observed_any_strict_complete_pass": false, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "assertion_failed": 4 } } } }, "HumanEval/23": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 107.75, "median": 100.0, "min": 90, "max": 141 }, "generated_token_seed_range": 51, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 111.25, "median": 114.0, "min": 53, "max": 164 }, "generated_token_seed_range": 111, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 150.5, "median": 147.5, "min": 116, "max": 191 }, "generated_token_seed_range": 75, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 50.5, "median": 50.5, "min": 48, "max": 53 }, "generated_token_seed_range": 5, "unique_trajectory_hashes": 2, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "code": { "ast_parse": 4, "executed": 4, "execution_statuses": { "passed": 4 } } } }, "gsm8k/test/1069": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 188.25, "median": 184.0, "min": 172, "max": 213 }, "generated_token_seed_range": 41, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 3, "fixed_budget_success_rate": 0.75, "strict_complete_success": 3, "strict_complete_success_rate": 0.75, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "300": 3, "100": 1 }, "modes": [ "300" ], "unique_absolute_majority": "300", "gold": "300" } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 193.5, "median": 185.5, "min": 175, "max": 228 }, "generated_token_seed_range": 53, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "300": 4 }, "modes": [ "300" ], "unique_absolute_majority": "300", "gold": "300" } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 219, "median": 218.0, "min": 186, "max": 254 }, "generated_token_seed_range": 68, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 3, "fixed_budget_success_rate": 0.75, "strict_complete_success": 3, "strict_complete_success_rate": 0.75, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "100": 1, "300": 3 }, "modes": [ "300" ], "unique_absolute_majority": "300", "gold": "300" } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 153.5, "median": 149.5, "min": 141, "max": 174 }, "generated_token_seed_range": 33, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "300": 4 }, "modes": [ "300" ], "unique_absolute_majority": "300", "gold": "300" } } }, "gsm8k/test/1228": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 229, "median": 225.5, "min": 215, "max": 250 }, "generated_token_seed_range": 35, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "31": 4 }, "modes": [ "31" ], "unique_absolute_majority": "31", "gold": "31" } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 253.25, "median": 255.5, "min": 243, "max": 259 }, "generated_token_seed_range": 16, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "31": 4 }, "modes": [ "31" ], "unique_absolute_majority": "31", "gold": "31" } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 151.25, "median": 135.0, "min": 72, "max": 263 }, "generated_token_seed_range": 191, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "31": 4 }, "modes": [ "31" ], "unique_absolute_majority": "31", "gold": "31" } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 193.5, "median": 223.0, "min": 72, "max": 256 }, "generated_token_seed_range": 184, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "31": 4 }, "modes": [ "31" ], "unique_absolute_majority": "31", "gold": "31" } } }, "gsm8k/test/0144": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 228, "median": 238.5, "min": 190, "max": 245 }, "generated_token_seed_range": 55, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 2, "fixed_budget_success_rate": 0.5, "strict_complete_success": 2, "strict_complete_success_rate": 0.5, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "20": 2, "36": 1, "140": 1 }, "modes": [ "20" ], "unique_absolute_majority": null, "gold": "20" } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 219, "median": 219.0, "min": 211, "max": 227 }, "generated_token_seed_range": 16, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 2, "fixed_budget_success_rate": 0.5, "strict_complete_success": 2, "strict_complete_success_rate": 0.5, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "20": 2, "36": 1, "140": 1 }, "modes": [ "20" ], "unique_absolute_majority": null, "gold": "20" } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 249.75, "median": 244.5, "min": 150, "max": 360 }, "generated_token_seed_range": 210, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 3, "fixed_budget_success_rate": 0.75, "strict_complete_success": 3, "strict_complete_success_rate": 0.75, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "20": 3, "36": 1 }, "modes": [ "20" ], "unique_absolute_majority": "20", "gold": "20" } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 215, "median": 211.0, "min": 201, "max": 237 }, "generated_token_seed_range": 36, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 1, "fixed_budget_success_rate": 0.25, "strict_complete_success": 1, "strict_complete_success_rate": 0.25, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "92": 2, "20": 1, "36": 1 }, "modes": [ "92" ], "unique_absolute_majority": null, "gold": "20" } } }, "gsm8k/test/1251": { "s0_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 272, "median": 265.0, "min": 253, "max": 305 }, "generated_token_seed_range": 52, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 1, "fixed_budget_success_rate": 0.25, "strict_complete_success": 1, "strict_complete_success_rate": 0.25, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "31.25": 1, "42.5": 2, "30": 1 }, "modes": [ "42.5" ], "unique_absolute_majority": null, "gold": "30" } }, "s1_eos": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 199, "median": 190.0, "min": 169, "max": 247 }, "generated_token_seed_range": 78, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 4, "fixed_budget_success_rate": 1, "strict_complete_success": 4, "strict_complete_success_rate": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 0, "math_answers": { "frequencies": { "30": 4 }, "modes": [ "30" ], "unique_absolute_majority": "30", "gold": "30" } }, "s0_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 220, "median": 220.0, "min": 183, "max": 257 }, "generated_token_seed_range": 74, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 1, "fixed_budget_success_rate": 0.25, "strict_complete_success": 1, "strict_complete_success_rate": 0.25, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "30": 1, "6": 2, "40": 1 }, "modes": [ "6" ], "unique_absolute_majority": null, "gold": "30" } }, "s1_period": { "outputs": 4, "natural_eos": 4, "natural_eos_rate": 1, "budget_truncated": 0, "generated_tokens": { "count": 4, "mean": 222, "median": 232.0, "min": 182, "max": 242 }, "generated_token_seed_range": 60, "unique_trajectory_hashes": 4, "completion_classes": { "NATURAL_EOS": 4 }, "fixed_budget_success": 3, "fixed_budget_success_rate": 0.75, "strict_complete_success": 3, "strict_complete_success_rate": 0.75, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "seed_success_range": 1, "math_answers": { "frequencies": { "30": 3, "6": 1 }, "modes": [ "30" ], "unique_absolute_majority": "30", "gold": "30" } } } }, "domain_condition_summary": { "english": { "s0_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 15, "natural_eos_rate": 0.9375, "budget_truncated": 1, "mean_generated_tokens": 316.8125, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 0.9375, "median": 1.0, "min": 0.75, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 316.8125, "median": 321.0, "min": 220.5, "max": 404.75 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 231.5, "median": 193.5, "min": 152, "max": 387 }, "between_source_mean_length_range": 184.25, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } }, "s1_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 249.75, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 249.75, "median": 257.125, "min": 64, "max": 420.75 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 113.75, "median": 143.5, "min": 11, "max": 157 }, "between_source_mean_length_range": 356.75, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } }, "s0_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 13, "natural_eos_rate": 0.8125, "budget_truncated": 3, "mean_generated_tokens": 378.1875, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 0.8125, "median": 0.875, "min": 0.5, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 378.1875, "median": 383.625, "min": 277.5, "max": 468 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 184.25, "median": 145.5, "min": 54, "max": 392 }, "between_source_mean_length_range": 190.5, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } }, "s1_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 171.3125, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 171.3125, "median": 179.375, "min": 61.75, "max": 264.75 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 80.75, "median": 61.5, "min": 30, "max": 170 }, "between_source_mean_length_range": 203.0, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } } }, "chinese": { "s0_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 157.8125, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 157.8125, "median": 156.375, "min": 94.25, "max": 224.25 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 83, "median": 69.0, "min": 50, "max": 144 }, "between_source_mean_length_range": 130.0, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } }, "s1_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 170.0625, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 170.0625, "median": 160.75, "min": 96, "max": 262.75 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 153.75, "median": 104.0, "min": 44, "max": 363 }, "between_source_mean_length_range": 166.75, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } }, "s0_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 163.0625, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 163.0625, "median": 151.25, "min": 98, "max": 251.75 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 74.25, "median": 46.5, "min": 14, "max": 190 }, "between_source_mean_length_range": 153.75, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } }, "s1_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 137, "fixed_budget_success": null, "strict_complete_success": null }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 137.0, "median": 116.25, "min": 89.25, "max": 226.25 }, "fixed_budget_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "strict_complete_success_rate": { "count": 0, "mean": null, "median": null, "min": null, "max": null } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 78.25, "median": 48.0, "min": 28, "max": 189 }, "between_source_mean_length_range": 137.0, "within_source_seed_success_range": { "count": 0, "mean": null, "median": null, "min": null, "max": null }, "between_source_success_rate_range": null } } }, "code": { "s0_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 355, "fixed_budget_success": 14, "strict_complete_success": 14 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 355.0, "median": 418.875, "min": 107.75, "max": 474.5 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.875, "median": 1.0, "min": 0.5, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.875, "median": 1.0, "min": 0.5, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 67.75, "median": 61.0, "min": 38, "max": 111 }, "between_source_mean_length_range": 366.75, "within_source_seed_success_range": { "count": 4, "mean": 0.25, "median": 0.0, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.5 } }, "s1_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 15, "natural_eos_rate": 0.9375, "budget_truncated": 1, "mean_generated_tokens": 323.1875, "fixed_budget_success": 13, "strict_complete_success": 13 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 0.9375, "median": 1.0, "min": 0.75, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 323.1875, "median": 374.0, "min": 111.25, "max": 433.5 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.8125, "median": 1.0, "min": 0.25, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.8125, "median": 1.0, "min": 0.25, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 88.75, "median": 97.0, "min": 21, "max": 140 }, "between_source_mean_length_range": 322.25, "within_source_seed_success_range": { "count": 4, "mean": 0.25, "median": 0.0, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.75 } }, "s0_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 15, "natural_eos_rate": 0.9375, "budget_truncated": 1, "mean_generated_tokens": 336.5625, "fixed_budget_success": 13, "strict_complete_success": 13 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 0.9375, "median": 1.0, "min": 0.75, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 336.5625, "median": 381.625, "min": 150.5, "max": 432.5 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.8125, "median": 1.0, "min": 0.25, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.8125, "median": 1.0, "min": 0.25, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 107.75, "median": 76.0, "min": 33, "max": 246 }, "between_source_mean_length_range": 282.0, "within_source_seed_success_range": { "count": 4, "mean": 0.25, "median": 0.0, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.75 } }, "s1_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 81.6875, "fixed_budget_success": 12, "strict_complete_success": 12 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 81.6875, "median": 64.75, "min": 50.5, "max": 146.75 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.75, "median": 1.0, "min": 0, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.75, "median": 1.0, "min": 0, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 33, "median": 15.5, "min": 0, "max": 101 }, "between_source_mean_length_range": 96.25, "within_source_seed_success_range": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0 }, "between_source_success_rate_range": 1 } } }, "math": { "s0_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 229.3125, "fixed_budget_success": 10, "strict_complete_success": 10 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 229.3125, "median": 228.5, "min": 188.25, "max": 272 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.625, "median": 0.625, "min": 0.25, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.625, "median": 0.625, "min": 0.25, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 45.75, "median": 46.5, "min": 35, "max": 55 }, "between_source_mean_length_range": 83.75, "within_source_seed_success_range": { "count": 4, "mean": 0.75, "median": 1.0, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.75 } }, "s1_eos": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 216.1875, "fixed_budget_success": 14, "strict_complete_success": 14 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 216.1875, "median": 209.0, "min": 193.5, "max": 253.25 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.875, "median": 1.0, "min": 0.5, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.875, "median": 1.0, "min": 0.5, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 40.75, "median": 34.5, "min": 16, "max": 78 }, "between_source_mean_length_range": 59.75, "within_source_seed_success_range": { "count": 4, "mean": 0.25, "median": 0.0, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.5 } }, "s0_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 210, "fixed_budget_success": 11, "strict_complete_success": 11 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 210.0, "median": 219.5, "min": 151.25, "max": 249.75 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.6875, "median": 0.75, "min": 0.25, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.6875, "median": 0.75, "min": 0.25, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 135.75, "median": 132.5, "min": 68, "max": 210 }, "between_source_mean_length_range": 98.5, "within_source_seed_success_range": { "count": 4, "mean": 0.75, "median": 1.0, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.75 } }, "s1_period": { "sources": 4, "outputs": 16, "micro": { "natural_eos": 16, "natural_eos_rate": 1, "budget_truncated": 0, "mean_generated_tokens": 196, "fixed_budget_success": 12, "strict_complete_success": 12 }, "source_macro": { "natural_eos_rate": { "count": 4, "mean": 1, "median": 1.0, "min": 1, "max": 1 }, "mean_generated_tokens": { "count": 4, "mean": 196.0, "median": 204.25, "min": 153.5, "max": 222 }, "fixed_budget_success_rate": { "count": 4, "mean": 0.75, "median": 0.875, "min": 0.25, "max": 1 }, "strict_complete_success_rate": { "count": 4, "mean": 0.75, "median": 0.875, "min": 0.25, "max": 1 } }, "variability": { "within_source_seed_length_range": { "count": 4, "mean": 78.25, "median": 48.0, "min": 33, "max": 184 }, "between_source_mean_length_range": 68.5, "within_source_seed_success_range": { "count": 4, "mean": 0.5, "median": 0.5, "min": 0, "max": 1 }, "between_source_success_rate_range": 0.75 } } } }, "source_contrasts": { "wikitext2/raw-validation/0443": { "natural_eos_rate": { "system_main": 0.125, "boundary_main_period_minus_eos": -0.125, "interaction": 0.25, "period_minus_eos_s0": -0.25, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0.25 }, "mean_generated_tokens": { "system_main": -111.25, "boundary_main_period_minus_eos": -121.0, "interaction": -310.0, "period_minus_eos_s0": 34, "period_minus_eos_s1": -276.0, "system_on_minus_off_eos": 43.75, "system_on_minus_off_period": -266.25 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "wikitext2/raw-validation/0030": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -247.75, "boundary_main_period_minus_eos": 44.5, "interaction": -93.5, "period_minus_eos_s0": 91.25, "period_minus_eos_s1": -2.25, "system_on_minus_off_eos": -201, "system_on_minus_off_period": -294.5 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "wikitext2/raw-validation/2909": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -47.5, "boundary_main_period_minus_eos": 41.0, "interaction": -32.0, "period_minus_eos_s0": 57.0, "period_minus_eos_s1": 25, "system_on_minus_off_eos": -31.5, "system_on_minus_off_period": -63.5 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "wikitext2/raw-validation/2746": { "natural_eos_rate": { "system_main": 0.375, "boundary_main_period_minus_eos": -0.125, "interaction": 0.25, "period_minus_eos_s0": -0.25, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0.25, "system_on_minus_off_period": 0.5 }, "mean_generated_tokens": { "system_main": -141.375, "boundary_main_period_minus_eos": 1.375, "interaction": -123.75, "period_minus_eos_s0": 63.25, "period_minus_eos_s1": -60.5, "system_on_minus_off_eos": -79.5, "system_on_minus_off_period": -203.25 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "tnews/test/4855": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -5.375, "boundary_main_period_minus_eos": 0.375, "interaction": -14.25, "period_minus_eos_s0": 7.5, "period_minus_eos_s1": -6.75, "system_on_minus_off_eos": 1.75, "system_on_minus_off_period": -12.5 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "tnews/test/8935": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": 32.0, "boundary_main_period_minus_eos": -30.0, "interaction": -13.0, "period_minus_eos_s0": -23.5, "period_minus_eos_s1": -36.5, "system_on_minus_off_eos": 38.5, "system_on_minus_off_period": 25.5 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "tnews/test/3448": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": 22.75, "boundary_main_period_minus_eos": -7.5, "interaction": 27.5, "period_minus_eos_s0": -21.25, "period_minus_eos_s1": 6.25, "system_on_minus_off_eos": 9.0, "system_on_minus_off_period": 36.5 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "tnews/test/1059": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -77.0, "boundary_main_period_minus_eos": -18.5, "interaction": -153.5, "period_minus_eos_s0": 58.25, "period_minus_eos_s1": -95.25, "system_on_minus_off_eos": -0.25, "system_on_minus_off_period": -153.75 }, "fixed_budget_success_rate": null, "strict_complete_success_rate": null }, "HumanEval/31": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -213.375, "boundary_main_period_minus_eos": -178.625, "interaction": -295.25, "period_minus_eos_s0": -31, "period_minus_eos_s1": -326.25, "system_on_minus_off_eos": -65.75, "system_on_minus_off_period": -361 }, "fixed_budget_success_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "strict_complete_success_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 } }, "HumanEval/44": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0.5, "period_minus_eos_s0": -0.25, "period_minus_eos_s1": 0.25, "system_on_minus_off_eos": -0.25, "system_on_minus_off_period": 0.25 }, "mean_generated_tokens": { "system_main": -163.375, "boundary_main_period_minus_eos": -164.375, "interaction": -244.75, "period_minus_eos_s0": -42.0, "period_minus_eos_s1": -286.75, "system_on_minus_off_eos": -41.0, "system_on_minus_off_period": -285.75 }, "fixed_budget_success_rate": { "system_main": 0.25, "boundary_main_period_minus_eos": 0.25, "interaction": 1.0, "period_minus_eos_s0": -0.25, "period_minus_eos_s1": 0.75, "system_on_minus_off_eos": -0.25, "system_on_minus_off_period": 0.75 }, "strict_complete_success_rate": { "system_main": 0.25, "boundary_main_period_minus_eos": 0.25, "interaction": 1.0, "period_minus_eos_s0": -0.25, "period_minus_eos_s1": 0.75, "system_on_minus_off_eos": -0.25, "system_on_minus_off_period": 0.75 } }, "HumanEval/133": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -148.375, "boundary_main_period_minus_eos": -167.875, "interaction": -248.75, "period_minus_eos_s0": -43.5, "period_minus_eos_s1": -292.25, "system_on_minus_off_eos": -24.0, "system_on_minus_off_period": -272.75 }, "fixed_budget_success_rate": { "system_main": -0.5, "boundary_main_period_minus_eos": -0.5, "interaction": -1, "period_minus_eos_s0": 0, "period_minus_eos_s1": -1, "system_on_minus_off_eos": 0, "system_on_minus_off_period": -1 }, "strict_complete_success_rate": { "system_main": -0.5, "boundary_main_period_minus_eos": -0.5, "interaction": -1, "period_minus_eos_s0": 0, "period_minus_eos_s1": -1, "system_on_minus_off_eos": 0, "system_on_minus_off_period": -1 } }, "HumanEval/23": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -48.25, "boundary_main_period_minus_eos": -9.0, "interaction": -103.5, "period_minus_eos_s0": 42.75, "period_minus_eos_s1": -60.75, "system_on_minus_off_eos": 3.5, "system_on_minus_off_period": -100.0 }, "fixed_budget_success_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "strict_complete_success_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 } }, "gsm8k/test/1069": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -30.125, "boundary_main_period_minus_eos": -4.625, "interaction": -70.75, "period_minus_eos_s0": 30.75, "period_minus_eos_s1": -40.0, "system_on_minus_off_eos": 5.25, "system_on_minus_off_period": -65.5 }, "fixed_budget_success_rate": { "system_main": 0.25, "boundary_main_period_minus_eos": 0.0, "interaction": 0.0, "period_minus_eos_s0": 0.0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0.25, "system_on_minus_off_period": 0.25 }, "strict_complete_success_rate": { "system_main": 0.25, "boundary_main_period_minus_eos": 0.0, "interaction": 0.0, "period_minus_eos_s0": 0.0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0.25, "system_on_minus_off_period": 0.25 } }, "gsm8k/test/1228": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": 33.25, "boundary_main_period_minus_eos": -68.75, "interaction": 18.0, "period_minus_eos_s0": -77.75, "period_minus_eos_s1": -59.75, "system_on_minus_off_eos": 24.25, "system_on_minus_off_period": 42.25 }, "fixed_budget_success_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "strict_complete_success_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 } }, "gsm8k/test/0144": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -21.875, "boundary_main_period_minus_eos": 8.875, "interaction": -25.75, "period_minus_eos_s0": 21.75, "period_minus_eos_s1": -4, "system_on_minus_off_eos": -9, "system_on_minus_off_period": -34.75 }, "fixed_budget_success_rate": { "system_main": -0.25, "boundary_main_period_minus_eos": 0.0, "interaction": -0.5, "period_minus_eos_s0": 0.25, "period_minus_eos_s1": -0.25, "system_on_minus_off_eos": 0.0, "system_on_minus_off_period": -0.5 }, "strict_complete_success_rate": { "system_main": -0.25, "boundary_main_period_minus_eos": 0.0, "interaction": -0.5, "period_minus_eos_s0": 0.25, "period_minus_eos_s1": -0.25, "system_on_minus_off_eos": 0.0, "system_on_minus_off_period": -0.5 } }, "gsm8k/test/1251": { "natural_eos_rate": { "system_main": 0.0, "boundary_main_period_minus_eos": 0.0, "interaction": 0, "period_minus_eos_s0": 0, "period_minus_eos_s1": 0, "system_on_minus_off_eos": 0, "system_on_minus_off_period": 0 }, "mean_generated_tokens": { "system_main": -35.5, "boundary_main_period_minus_eos": -14.5, "interaction": 75, "period_minus_eos_s0": -52, "period_minus_eos_s1": 23, "system_on_minus_off_eos": -73, "system_on_minus_off_period": 2 }, "fixed_budget_success_rate": { "system_main": 0.625, "boundary_main_period_minus_eos": -0.125, "interaction": -0.25, "period_minus_eos_s0": 0.0, "period_minus_eos_s1": -0.25, "system_on_minus_off_eos": 0.75, "system_on_minus_off_period": 0.5 }, "strict_complete_success_rate": { "system_main": 0.625, "boundary_main_period_minus_eos": -0.125, "interaction": -0.25, "period_minus_eos_s0": 0.0, "period_minus_eos_s1": -0.25, "system_on_minus_off_eos": 0.75, "system_on_minus_off_period": 0.5 } } }, "domain_contrasts": { "english": { "natural_eos_rate": { "system_main": { "count": 4, "mean": 0.125, "median": 0.0625, "min": 0.0, "max": 0.375, "directions": { "positive": 2, "zero": 2, "negative": 0 }, "by_source": { "wikitext2/raw-validation/0443": 0.125, "wikitext2/raw-validation/0030": 0.0, "wikitext2/raw-validation/2909": 0.0, "wikitext2/raw-validation/2746": 0.375 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -0.0625, "median": -0.0625, "min": -0.125, "max": 0.0, "directions": { "positive": 0, "zero": 2, "negative": 2 }, "by_source": { "wikitext2/raw-validation/0443": -0.125, "wikitext2/raw-validation/0030": 0.0, "wikitext2/raw-validation/2909": 0.0, "wikitext2/raw-validation/2746": -0.125 } }, "interaction": { "count": 4, "mean": 0.125, "median": 0.125, "min": 0, "max": 0.25, "directions": { "positive": 2, "zero": 2, "negative": 0 }, "by_source": { "wikitext2/raw-validation/0443": 0.25, "wikitext2/raw-validation/0030": 0, "wikitext2/raw-validation/2909": 0, "wikitext2/raw-validation/2746": 0.25 } }, "period_minus_eos_s0": { "count": 4, "mean": -0.125, "median": -0.125, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 2, "negative": 2 }, "by_source": { "wikitext2/raw-validation/0443": -0.25, "wikitext2/raw-validation/0030": 0, "wikitext2/raw-validation/2909": 0, "wikitext2/raw-validation/2746": -0.25 } }, "period_minus_eos_s1": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "wikitext2/raw-validation/0443": 0, "wikitext2/raw-validation/0030": 0, "wikitext2/raw-validation/2909": 0, "wikitext2/raw-validation/2746": 0 } }, "system_on_minus_off_eos": { "count": 4, "mean": 0.0625, "median": 0.0, "min": 0, "max": 0.25, "directions": { "positive": 1, "zero": 3, "negative": 0 }, "by_source": { "wikitext2/raw-validation/0443": 0, "wikitext2/raw-validation/0030": 0, "wikitext2/raw-validation/2909": 0, "wikitext2/raw-validation/2746": 0.25 } }, "system_on_minus_off_period": { "count": 4, "mean": 0.1875, "median": 0.125, "min": 0, "max": 0.5, "directions": { "positive": 2, "zero": 2, "negative": 0 }, "by_source": { "wikitext2/raw-validation/0443": 0.25, "wikitext2/raw-validation/0030": 0, "wikitext2/raw-validation/2909": 0, "wikitext2/raw-validation/2746": 0.5 } } }, "mean_generated_tokens": { "system_main": { "count": 4, "mean": -136.96875, "median": -126.3125, "min": -247.75, "max": -47.5, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "wikitext2/raw-validation/0443": -111.25, "wikitext2/raw-validation/0030": -247.75, "wikitext2/raw-validation/2909": -47.5, "wikitext2/raw-validation/2746": -141.375 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -8.53125, "median": 21.1875, "min": -121.0, "max": 44.5, "directions": { "positive": 3, "zero": 0, "negative": 1 }, "by_source": { "wikitext2/raw-validation/0443": -121.0, "wikitext2/raw-validation/0030": 44.5, "wikitext2/raw-validation/2909": 41.0, "wikitext2/raw-validation/2746": 1.375 } }, "interaction": { "count": 4, "mean": -139.8125, "median": -108.625, "min": -310.0, "max": -32.0, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "wikitext2/raw-validation/0443": -310.0, "wikitext2/raw-validation/0030": -93.5, "wikitext2/raw-validation/2909": -32.0, "wikitext2/raw-validation/2746": -123.75 } }, "period_minus_eos_s0": { "count": 4, "mean": 61.375, "median": 60.125, "min": 34, "max": 91.25, "directions": { "positive": 4, "zero": 0, "negative": 0 }, "by_source": { "wikitext2/raw-validation/0443": 34, "wikitext2/raw-validation/0030": 91.25, "wikitext2/raw-validation/2909": 57.0, "wikitext2/raw-validation/2746": 63.25 } }, "period_minus_eos_s1": { "count": 4, "mean": -78.4375, "median": -31.375, "min": -276.0, "max": 25, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "wikitext2/raw-validation/0443": -276.0, "wikitext2/raw-validation/0030": -2.25, "wikitext2/raw-validation/2909": 25, "wikitext2/raw-validation/2746": -60.5 } }, "system_on_minus_off_eos": { "count": 4, "mean": -67.0625, "median": -55.5, "min": -201, "max": 43.75, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "wikitext2/raw-validation/0443": 43.75, "wikitext2/raw-validation/0030": -201, "wikitext2/raw-validation/2909": -31.5, "wikitext2/raw-validation/2746": -79.5 } }, "system_on_minus_off_period": { "count": 4, "mean": -206.875, "median": -234.75, "min": -294.5, "max": -63.5, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "wikitext2/raw-validation/0443": -266.25, "wikitext2/raw-validation/0030": -294.5, "wikitext2/raw-validation/2909": -63.5, "wikitext2/raw-validation/2746": -203.25 } } }, "fixed_budget_success_rate": { "system_main": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "boundary_main_period_minus_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "interaction": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s0": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s1": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_period": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} } }, "strict_complete_success_rate": { "system_main": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "boundary_main_period_minus_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "interaction": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s0": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s1": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_period": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} } } }, "chinese": { "natural_eos_rate": { "system_main": { "count": 4, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0.0, "tnews/test/8935": 0.0, "tnews/test/3448": 0.0, "tnews/test/1059": 0.0 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0.0, "tnews/test/8935": 0.0, "tnews/test/3448": 0.0, "tnews/test/1059": 0.0 } }, "interaction": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0, "tnews/test/8935": 0, "tnews/test/3448": 0, "tnews/test/1059": 0 } }, "period_minus_eos_s0": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0, "tnews/test/8935": 0, "tnews/test/3448": 0, "tnews/test/1059": 0 } }, "period_minus_eos_s1": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0, "tnews/test/8935": 0, "tnews/test/3448": 0, "tnews/test/1059": 0 } }, "system_on_minus_off_eos": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0, "tnews/test/8935": 0, "tnews/test/3448": 0, "tnews/test/1059": 0 } }, "system_on_minus_off_period": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "tnews/test/4855": 0, "tnews/test/8935": 0, "tnews/test/3448": 0, "tnews/test/1059": 0 } } }, "mean_generated_tokens": { "system_main": { "count": 4, "mean": -6.90625, "median": 8.6875, "min": -77.0, "max": 32.0, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "tnews/test/4855": -5.375, "tnews/test/8935": 32.0, "tnews/test/3448": 22.75, "tnews/test/1059": -77.0 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -13.90625, "median": -13.0, "min": -30.0, "max": 0.375, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "tnews/test/4855": 0.375, "tnews/test/8935": -30.0, "tnews/test/3448": -7.5, "tnews/test/1059": -18.5 } }, "interaction": { "count": 4, "mean": -38.3125, "median": -13.625, "min": -153.5, "max": 27.5, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "tnews/test/4855": -14.25, "tnews/test/8935": -13.0, "tnews/test/3448": 27.5, "tnews/test/1059": -153.5 } }, "period_minus_eos_s0": { "count": 4, "mean": 5.25, "median": -6.875, "min": -23.5, "max": 58.25, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "tnews/test/4855": 7.5, "tnews/test/8935": -23.5, "tnews/test/3448": -21.25, "tnews/test/1059": 58.25 } }, "period_minus_eos_s1": { "count": 4, "mean": -33.0625, "median": -21.625, "min": -95.25, "max": 6.25, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "tnews/test/4855": -6.75, "tnews/test/8935": -36.5, "tnews/test/3448": 6.25, "tnews/test/1059": -95.25 } }, "system_on_minus_off_eos": { "count": 4, "mean": 12.25, "median": 5.375, "min": -0.25, "max": 38.5, "directions": { "positive": 3, "zero": 0, "negative": 1 }, "by_source": { "tnews/test/4855": 1.75, "tnews/test/8935": 38.5, "tnews/test/3448": 9.0, "tnews/test/1059": -0.25 } }, "system_on_minus_off_period": { "count": 4, "mean": -26.0625, "median": 6.5, "min": -153.75, "max": 36.5, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "tnews/test/4855": -12.5, "tnews/test/8935": 25.5, "tnews/test/3448": 36.5, "tnews/test/1059": -153.75 } } }, "fixed_budget_success_rate": { "system_main": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "boundary_main_period_minus_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "interaction": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s0": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s1": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_period": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} } }, "strict_complete_success_rate": { "system_main": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "boundary_main_period_minus_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "interaction": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s0": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "period_minus_eos_s1": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_eos": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} }, "system_on_minus_off_period": { "count": 0, "mean": null, "median": null, "min": null, "max": null, "directions": { "positive": 0, "zero": 0, "negative": 0 }, "by_source": {} } } }, "code": { "natural_eos_rate": { "system_main": { "count": 4, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "HumanEval/31": 0.0, "HumanEval/44": 0.0, "HumanEval/133": 0.0, "HumanEval/23": 0.0 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "HumanEval/31": 0.0, "HumanEval/44": 0.0, "HumanEval/133": 0.0, "HumanEval/23": 0.0 } }, "interaction": { "count": 4, "mean": 0.125, "median": 0.0, "min": 0, "max": 0.5, "directions": { "positive": 1, "zero": 3, "negative": 0 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.5, "HumanEval/133": 0, "HumanEval/23": 0 } }, "period_minus_eos_s0": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": -0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "period_minus_eos_s1": { "count": 4, "mean": 0.0625, "median": 0.0, "min": 0, "max": 0.25, "directions": { "positive": 1, "zero": 3, "negative": 0 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "system_on_minus_off_eos": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": -0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "system_on_minus_off_period": { "count": 4, "mean": 0.0625, "median": 0.0, "min": 0, "max": 0.25, "directions": { "positive": 1, "zero": 3, "negative": 0 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.25, "HumanEval/133": 0, "HumanEval/23": 0 } } }, "mean_generated_tokens": { "system_main": { "count": 4, "mean": -143.34375, "median": -155.875, "min": -213.375, "max": -48.25, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "HumanEval/31": -213.375, "HumanEval/44": -163.375, "HumanEval/133": -148.375, "HumanEval/23": -48.25 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -129.96875, "median": -166.125, "min": -178.625, "max": -9.0, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "HumanEval/31": -178.625, "HumanEval/44": -164.375, "HumanEval/133": -167.875, "HumanEval/23": -9.0 } }, "interaction": { "count": 4, "mean": -223.0625, "median": -246.75, "min": -295.25, "max": -103.5, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "HumanEval/31": -295.25, "HumanEval/44": -244.75, "HumanEval/133": -248.75, "HumanEval/23": -103.5 } }, "period_minus_eos_s0": { "count": 4, "mean": -18.4375, "median": -36.5, "min": -43.5, "max": 42.75, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "HumanEval/31": -31, "HumanEval/44": -42.0, "HumanEval/133": -43.5, "HumanEval/23": 42.75 } }, "period_minus_eos_s1": { "count": 4, "mean": -241.5, "median": -289.5, "min": -326.25, "max": -60.75, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "HumanEval/31": -326.25, "HumanEval/44": -286.75, "HumanEval/133": -292.25, "HumanEval/23": -60.75 } }, "system_on_minus_off_eos": { "count": 4, "mean": -31.8125, "median": -32.5, "min": -65.75, "max": 3.5, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "HumanEval/31": -65.75, "HumanEval/44": -41.0, "HumanEval/133": -24.0, "HumanEval/23": 3.5 } }, "system_on_minus_off_period": { "count": 4, "mean": -254.875, "median": -279.25, "min": -361, "max": -100.0, "directions": { "positive": 0, "zero": 0, "negative": 4 }, "by_source": { "HumanEval/31": -361, "HumanEval/44": -285.75, "HumanEval/133": -272.75, "HumanEval/23": -100.0 } } }, "fixed_budget_success_rate": { "system_main": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.5, "max": 0.25, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0.0, "HumanEval/44": 0.25, "HumanEval/133": -0.5, "HumanEval/23": 0.0 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.5, "max": 0.25, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0.0, "HumanEval/44": 0.25, "HumanEval/133": -0.5, "HumanEval/23": 0.0 } }, "interaction": { "count": 4, "mean": 0.0, "median": 0.0, "min": -1, "max": 1.0, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 1.0, "HumanEval/133": -1, "HumanEval/23": 0 } }, "period_minus_eos_s0": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": -0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "period_minus_eos_s1": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -1, "max": 0.75, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.75, "HumanEval/133": -1, "HumanEval/23": 0 } }, "system_on_minus_off_eos": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": -0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "system_on_minus_off_period": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -1, "max": 0.75, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.75, "HumanEval/133": -1, "HumanEval/23": 0 } } }, "strict_complete_success_rate": { "system_main": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.5, "max": 0.25, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0.0, "HumanEval/44": 0.25, "HumanEval/133": -0.5, "HumanEval/23": 0.0 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.5, "max": 0.25, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0.0, "HumanEval/44": 0.25, "HumanEval/133": -0.5, "HumanEval/23": 0.0 } }, "interaction": { "count": 4, "mean": 0.0, "median": 0.0, "min": -1, "max": 1.0, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 1.0, "HumanEval/133": -1, "HumanEval/23": 0 } }, "period_minus_eos_s0": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": -0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "period_minus_eos_s1": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -1, "max": 0.75, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.75, "HumanEval/133": -1, "HumanEval/23": 0 } }, "system_on_minus_off_eos": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": -0.25, "HumanEval/133": 0, "HumanEval/23": 0 } }, "system_on_minus_off_period": { "count": 4, "mean": -0.0625, "median": 0.0, "min": -1, "max": 0.75, "directions": { "positive": 1, "zero": 2, "negative": 1 }, "by_source": { "HumanEval/31": 0, "HumanEval/44": 0.75, "HumanEval/133": -1, "HumanEval/23": 0 } } } }, "math": { "natural_eos_rate": { "system_main": { "count": 4, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0.0, "gsm8k/test/0144": 0.0, "gsm8k/test/1251": 0.0 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0.0, "gsm8k/test/0144": 0.0, "gsm8k/test/1251": 0.0 } }, "interaction": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0, "gsm8k/test/1251": 0 } }, "period_minus_eos_s0": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0, "gsm8k/test/1251": 0 } }, "period_minus_eos_s1": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0, "gsm8k/test/1251": 0 } }, "system_on_minus_off_eos": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0, "gsm8k/test/1251": 0 } }, "system_on_minus_off_period": { "count": 4, "mean": 0, "median": 0.0, "min": 0, "max": 0, "directions": { "positive": 0, "zero": 4, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0, "gsm8k/test/1251": 0 } } }, "mean_generated_tokens": { "system_main": { "count": 4, "mean": -13.5625, "median": -26.0, "min": -35.5, "max": 33.25, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "gsm8k/test/1069": -30.125, "gsm8k/test/1228": 33.25, "gsm8k/test/0144": -21.875, "gsm8k/test/1251": -35.5 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -19.75, "median": -9.5625, "min": -68.75, "max": 8.875, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "gsm8k/test/1069": -4.625, "gsm8k/test/1228": -68.75, "gsm8k/test/0144": 8.875, "gsm8k/test/1251": -14.5 } }, "interaction": { "count": 4, "mean": -0.875, "median": -3.875, "min": -70.75, "max": 75, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "gsm8k/test/1069": -70.75, "gsm8k/test/1228": 18.0, "gsm8k/test/0144": -25.75, "gsm8k/test/1251": 75 } }, "period_minus_eos_s0": { "count": 4, "mean": -19.3125, "median": -15.125, "min": -77.75, "max": 30.75, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "gsm8k/test/1069": 30.75, "gsm8k/test/1228": -77.75, "gsm8k/test/0144": 21.75, "gsm8k/test/1251": -52 } }, "period_minus_eos_s1": { "count": 4, "mean": -20.1875, "median": -22.0, "min": -59.75, "max": 23, "directions": { "positive": 1, "zero": 0, "negative": 3 }, "by_source": { "gsm8k/test/1069": -40.0, "gsm8k/test/1228": -59.75, "gsm8k/test/0144": -4, "gsm8k/test/1251": 23 } }, "system_on_minus_off_eos": { "count": 4, "mean": -13.125, "median": -1.875, "min": -73, "max": 24.25, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "gsm8k/test/1069": 5.25, "gsm8k/test/1228": 24.25, "gsm8k/test/0144": -9, "gsm8k/test/1251": -73 } }, "system_on_minus_off_period": { "count": 4, "mean": -14.0, "median": -16.375, "min": -65.5, "max": 42.25, "directions": { "positive": 2, "zero": 0, "negative": 2 }, "by_source": { "gsm8k/test/1069": -65.5, "gsm8k/test/1228": 42.25, "gsm8k/test/0144": -34.75, "gsm8k/test/1251": 2 } } }, "fixed_budget_success_rate": { "system_main": { "count": 4, "mean": 0.15625, "median": 0.125, "min": -0.25, "max": 0.625, "directions": { "positive": 2, "zero": 1, "negative": 1 }, "by_source": { "gsm8k/test/1069": 0.25, "gsm8k/test/1228": 0.0, "gsm8k/test/0144": -0.25, "gsm8k/test/1251": 0.625 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -0.03125, "median": 0.0, "min": -0.125, "max": 0.0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0.0, "gsm8k/test/0144": 0.0, "gsm8k/test/1251": -0.125 } }, "interaction": { "count": 4, "mean": -0.1875, "median": -0.125, "min": -0.5, "max": 0.0, "directions": { "positive": 0, "zero": 2, "negative": 2 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0, "gsm8k/test/0144": -0.5, "gsm8k/test/1251": -0.25 } }, "period_minus_eos_s0": { "count": 4, "mean": 0.0625, "median": 0.0, "min": 0.0, "max": 0.25, "directions": { "positive": 1, "zero": 3, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0.25, "gsm8k/test/1251": 0.0 } }, "period_minus_eos_s1": { "count": 4, "mean": -0.125, "median": -0.125, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 2, "negative": 2 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": -0.25, "gsm8k/test/1251": -0.25 } }, "system_on_minus_off_eos": { "count": 4, "mean": 0.25, "median": 0.125, "min": 0, "max": 0.75, "directions": { "positive": 2, "zero": 2, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0.25, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0.0, "gsm8k/test/1251": 0.75 } }, "system_on_minus_off_period": { "count": 4, "mean": 0.0625, "median": 0.125, "min": -0.5, "max": 0.5, "directions": { "positive": 2, "zero": 1, "negative": 1 }, "by_source": { "gsm8k/test/1069": 0.25, "gsm8k/test/1228": 0, "gsm8k/test/0144": -0.5, "gsm8k/test/1251": 0.5 } } }, "strict_complete_success_rate": { "system_main": { "count": 4, "mean": 0.15625, "median": 0.125, "min": -0.25, "max": 0.625, "directions": { "positive": 2, "zero": 1, "negative": 1 }, "by_source": { "gsm8k/test/1069": 0.25, "gsm8k/test/1228": 0.0, "gsm8k/test/0144": -0.25, "gsm8k/test/1251": 0.625 } }, "boundary_main_period_minus_eos": { "count": 4, "mean": -0.03125, "median": 0.0, "min": -0.125, "max": 0.0, "directions": { "positive": 0, "zero": 3, "negative": 1 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0.0, "gsm8k/test/0144": 0.0, "gsm8k/test/1251": -0.125 } }, "interaction": { "count": 4, "mean": -0.1875, "median": -0.125, "min": -0.5, "max": 0.0, "directions": { "positive": 0, "zero": 2, "negative": 2 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0, "gsm8k/test/0144": -0.5, "gsm8k/test/1251": -0.25 } }, "period_minus_eos_s0": { "count": 4, "mean": 0.0625, "median": 0.0, "min": 0.0, "max": 0.25, "directions": { "positive": 1, "zero": 3, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0.0, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0.25, "gsm8k/test/1251": 0.0 } }, "period_minus_eos_s1": { "count": 4, "mean": -0.125, "median": -0.125, "min": -0.25, "max": 0, "directions": { "positive": 0, "zero": 2, "negative": 2 }, "by_source": { "gsm8k/test/1069": 0, "gsm8k/test/1228": 0, "gsm8k/test/0144": -0.25, "gsm8k/test/1251": -0.25 } }, "system_on_minus_off_eos": { "count": 4, "mean": 0.25, "median": 0.125, "min": 0, "max": 0.75, "directions": { "positive": 2, "zero": 2, "negative": 0 }, "by_source": { "gsm8k/test/1069": 0.25, "gsm8k/test/1228": 0, "gsm8k/test/0144": 0.0, "gsm8k/test/1251": 0.75 } }, "system_on_minus_off_period": { "count": 4, "mean": 0.0625, "median": 0.125, "min": -0.5, "max": 0.5, "directions": { "positive": 2, "zero": 1, "negative": 1 }, "by_source": { "gsm8k/test/1069": 0.25, "gsm8k/test/1228": 0, "gsm8k/test/0144": -0.5, "gsm8k/test/1251": 0.5 } } } } }, "task_matrix": { "code": [ { "source_id": "HumanEval/31", "within_domain_index": 0, "conditions": { "s0_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 451, "median": 467.5, "min": 379, "max": 490 } }, "s1_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 385.25, "median": 379.0, "min": 350, "max": 433 } }, "s0_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 420, "median": 419.5, "min": 382, "max": 459 } }, "s1_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 59, "median": 59.0, "min": 59, "max": 59 } } } }, { "source_id": "HumanEval/44", "within_domain_index": 1, "conditions": { "s0_eos": { "fixed_budget_success": 2, "strict_complete_success": 2, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 474.5, "median": 476.5, "min": 437, "max": 508 } }, "s1_eos": { "fixed_budget_success": 1, "strict_complete_success": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 3, "generated_tokens": { "count": 4, "mean": 433.5, "median": 425.0, "min": 372, "max": 512 } }, "s0_period": { "fixed_budget_success": 1, "strict_complete_success": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 3, "generated_tokens": { "count": 4, "mean": 432.5, "median": 476.0, "min": 266, "max": 512 } }, "s1_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 146.75, "median": 164.0, "min": 79, "max": 180 } } } }, { "source_id": "HumanEval/133", "within_domain_index": 2, "conditions": { "s0_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 386.75, "median": 388.5, "min": 366, "max": 404 } }, "s1_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 362.75, "median": 360.0, "min": 355, "max": 376 } }, "s0_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 343.25, "median": 341.0, "min": 329, "max": 362 } }, "s1_period": { "fixed_budget_success": 0, "strict_complete_success": 0, "observed_any_fixed_budget_pass": false, "observed_any_strict_complete_pass": false, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 70.5, "median": 76.0, "min": 52, "max": 78 } } } }, { "source_id": "HumanEval/23", "within_domain_index": 3, "conditions": { "s0_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 107.75, "median": 100.0, "min": 90, "max": 141 } }, "s1_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 111.25, "median": 114.0, "min": 53, "max": 164 } }, "s0_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 150.5, "median": 147.5, "min": 116, "max": 191 } }, "s1_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 50.5, "median": 50.5, "min": 48, "max": 53 } } } } ], "math": [ { "source_id": "gsm8k/test/1069", "within_domain_index": 0, "conditions": { "s0_eos": { "fixed_budget_success": 3, "strict_complete_success": 3, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 188.25, "median": 184.0, "min": 172, "max": 213 } }, "s1_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 193.5, "median": 185.5, "min": 175, "max": 228 } }, "s0_period": { "fixed_budget_success": 3, "strict_complete_success": 3, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 219, "median": 218.0, "min": 186, "max": 254 } }, "s1_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 153.5, "median": 149.5, "min": 141, "max": 174 } } } }, { "source_id": "gsm8k/test/1228", "within_domain_index": 1, "conditions": { "s0_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 229, "median": 225.5, "min": 215, "max": 250 } }, "s1_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 253.25, "median": 255.5, "min": 243, "max": 259 } }, "s0_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 151.25, "median": 135.0, "min": 72, "max": 263 } }, "s1_period": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 193.5, "median": 223.0, "min": 72, "max": 256 } } } }, { "source_id": "gsm8k/test/0144", "within_domain_index": 2, "conditions": { "s0_eos": { "fixed_budget_success": 2, "strict_complete_success": 2, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 228, "median": 238.5, "min": 190, "max": 245 } }, "s1_eos": { "fixed_budget_success": 2, "strict_complete_success": 2, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 219, "median": 219.0, "min": 211, "max": 227 } }, "s0_period": { "fixed_budget_success": 3, "strict_complete_success": 3, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 249.75, "median": 244.5, "min": 150, "max": 360 } }, "s1_period": { "fixed_budget_success": 1, "strict_complete_success": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 215, "median": 211.0, "min": 201, "max": 237 } } } }, { "source_id": "gsm8k/test/1251", "within_domain_index": 3, "conditions": { "s0_eos": { "fixed_budget_success": 1, "strict_complete_success": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 272, "median": 265.0, "min": 253, "max": 305 } }, "s1_eos": { "fixed_budget_success": 4, "strict_complete_success": 4, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 199, "median": 190.0, "min": 169, "max": 247 } }, "s0_period": { "fixed_budget_success": 1, "strict_complete_success": 1, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 220, "median": 220.0, "min": 183, "max": 257 } }, "s1_period": { "fixed_budget_success": 3, "strict_complete_success": 3, "observed_any_fixed_budget_pass": true, "observed_any_strict_complete_pass": true, "natural_eos": 4, "generated_tokens": { "count": 4, "mean": 222, "median": 232.0, "min": 182, "max": 242 } } } } ] }, "prior_direction_check": { "english": { "period_shortens_mean_tokens": { "by_source": { "wikitext2/raw-validation/0443": true, "wikitext2/raw-validation/0030": false, "wikitext2/raw-validation/2909": false, "wikitext2/raw-validation/2746": false } }, "system_on_raises_natural_eos_rate": { "by_source": { "wikitext2/raw-validation/0443": true, "wikitext2/raw-validation/0030": false, "wikitext2/raw-validation/2909": false, "wikitext2/raw-validation/2746": true } } }, "chinese": { "period_shortens_mean_tokens": { "by_source": { "tnews/test/4855": false, "tnews/test/8935": true, "tnews/test/3448": true, "tnews/test/1059": true } }, "system_on_raises_natural_eos_rate": { "by_source": { "tnews/test/4855": false, "tnews/test/8935": false, "tnews/test/3448": false, "tnews/test/1059": false } } }, "code": { "period_shortens_mean_tokens": { "by_source": { "HumanEval/31": true, "HumanEval/44": true, "HumanEval/133": true, "HumanEval/23": true } }, "system_on_raises_natural_eos_rate": { "by_source": { "HumanEval/31": false, "HumanEval/44": false, "HumanEval/133": false, "HumanEval/23": false } } }, "math": { "period_shortens_mean_tokens": { "by_source": { "gsm8k/test/1069": true, "gsm8k/test/1228": true, "gsm8k/test/0144": false, "gsm8k/test/1251": true } }, "system_on_raises_natural_eos_rate": { "by_source": { "gsm8k/test/1069": false, "gsm8k/test/1228": false, "gsm8k/test/0144": false, "gsm8k/test/1251": false } } } }, "claim_boundary": [ "Four sources per domain are not full benchmark estimates.", "Seed-level repeats are not independent task observations.", "Source-blocked contrasts are descriptive and have no p-values.", "Observed any-pass is not standard HumanEval pass@4.", "Natural EOS, evaluator coverage, and correctness remain separate.", "The period condition is a counterfactual, not an official-valid chat.", "Round 06 motivated the period contrast, so this is a directed follow-up." ], "content_hash": "9c957e3bc468931c7d0b38ae3285fa5509015d3feb7cf9d4a61ce4c6b5c2f64b" }