wuyang
6079 lines
164 KiB
JSON
6079 lines
164 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"protocol_id": "llm-atlas-deepseek-chat-cross-source-sampling-v1",
|
|
"inputs": {
|
|
"sampling": {
|
|
"path": "/tmp/deepseek-v2-lite-chat-cross-source-sampling-formal.json",
|
|
"sha256": "f013132485f27adce008f03f781bed9982efc0d7939f13faede01c9f6f3d7f7c",
|
|
"content_hash": "8fa14db0e9e6f1fd649a500796953ca948606a2c525d9c92c4b5f848beeaa3a7"
|
|
},
|
|
"evaluation": {
|
|
"path": "/tmp/deepseek-v2-lite-chat-cross-source-sampling-eval.json",
|
|
"sha256": "e88b274599fc5951561f9e5e7438fb4d6d25f341ae3bc4a2c4121877a0db8975",
|
|
"content_hash": "66326e89a92e4c3b477bd813ca9b7139a77312341cdd6b1ccc32fe557380d021"
|
|
},
|
|
"reproduction": {
|
|
"path": "/tmp/deepseek-v2-lite-chat-cross-source-sampling-reproduction.json",
|
|
"sha256": "ec4a47894953f5d73bb62211b588632ef7032da0e915218992b7fc3ef9c2a556",
|
|
"summary": {
|
|
"cells": 64,
|
|
"all_preregistered_fields_exact": 64,
|
|
"by_field": {
|
|
"run_seed_exact": 64,
|
|
"prompt_hash_exact": 64,
|
|
"generated_token_ids_exact": 64,
|
|
"decoded_text_exact": 64,
|
|
"eos_state_exact": 64,
|
|
"truncation_state_exact": 64,
|
|
"cpu_rng_pre_state_exact": 64,
|
|
"cuda_rng_pre_state_exact": 64
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"contract": {
|
|
"source_is_primary_coverage_unit": true,
|
|
"seed_is_within_source_repeat": true,
|
|
"sources": 16,
|
|
"sources_per_domain": 4,
|
|
"seeds_per_source_condition": 4,
|
|
"conditions": [
|
|
"s0_eos",
|
|
"s1_eos",
|
|
"s0_period",
|
|
"s1_period"
|
|
],
|
|
"outputs": 256,
|
|
"source_order": [
|
|
"wikitext2/raw-validation/0443",
|
|
"wikitext2/raw-validation/0030",
|
|
"wikitext2/raw-validation/2909",
|
|
"wikitext2/raw-validation/2746",
|
|
"tnews/test/4855",
|
|
"tnews/test/8935",
|
|
"tnews/test/3448",
|
|
"tnews/test/1059",
|
|
"HumanEval/31",
|
|
"HumanEval/44",
|
|
"HumanEval/133",
|
|
"HumanEval/23",
|
|
"gsm8k/test/1069",
|
|
"gsm8k/test/1228",
|
|
"gsm8k/test/0144",
|
|
"gsm8k/test/1251"
|
|
],
|
|
"sources_by_domain": {
|
|
"english": [
|
|
"wikitext2/raw-validation/0443",
|
|
"wikitext2/raw-validation/0030",
|
|
"wikitext2/raw-validation/2909",
|
|
"wikitext2/raw-validation/2746"
|
|
],
|
|
"chinese": [
|
|
"tnews/test/4855",
|
|
"tnews/test/8935",
|
|
"tnews/test/3448",
|
|
"tnews/test/1059"
|
|
],
|
|
"code": [
|
|
"HumanEval/31",
|
|
"HumanEval/44",
|
|
"HumanEval/133",
|
|
"HumanEval/23"
|
|
],
|
|
"math": [
|
|
"gsm8k/test/1069",
|
|
"gsm8k/test/1228",
|
|
"gsm8k/test/0144",
|
|
"gsm8k/test/1251"
|
|
]
|
|
},
|
|
"no_population_confidence_intervals": true,
|
|
"no_p_values": true
|
|
},
|
|
"overall": {
|
|
"outputs": 256,
|
|
"natural_eos": 250,
|
|
"budget_truncated": 6,
|
|
"unique_trajectory_hashes": 247,
|
|
"math": {
|
|
"outputs": 64,
|
|
"fixed_budget_success": 47,
|
|
"strict_complete_success": 47
|
|
},
|
|
"code": {
|
|
"outputs": 64,
|
|
"fixed_budget_success": 52,
|
|
"strict_complete_success": 52
|
|
}
|
|
},
|
|
"source_metadata": {
|
|
"wikitext2/raw-validation/0443": {
|
|
"domain": "english",
|
|
"within_domain_index": 0,
|
|
"selection_rank": "004b632f55ea14a7a1f99552db185df62bb0bbd0914ca94c411d0325d0e96639"
|
|
},
|
|
"wikitext2/raw-validation/0030": {
|
|
"domain": "english",
|
|
"within_domain_index": 1,
|
|
"selection_rank": "0064700bc7cd279cd9669445fce1a5a410d6e0cae091efed9c8f0182e47d7ab0"
|
|
},
|
|
"wikitext2/raw-validation/2909": {
|
|
"domain": "english",
|
|
"within_domain_index": 2,
|
|
"selection_rank": "00a0c873d028d892e454883e687d7af51aee2831fe2f7884668ca6dfe0d681c8"
|
|
},
|
|
"wikitext2/raw-validation/2746": {
|
|
"domain": "english",
|
|
"within_domain_index": 3,
|
|
"selection_rank": "00c7a9a1b242b365225d7fdeec777508a5a3d90a76a7cf4cd2e93cee8f1693f4"
|
|
},
|
|
"tnews/test/4855": {
|
|
"domain": "chinese",
|
|
"within_domain_index": 0,
|
|
"selection_rank": "000b4d139c925918017352dc899d44353839fb496e1a607335490e402857bf46"
|
|
},
|
|
"tnews/test/8935": {
|
|
"domain": "chinese",
|
|
"within_domain_index": 1,
|
|
"selection_rank": "000c9943109217e6b0473de9c090009fae3ee782aa8e3c63f08abe82cb8c487e"
|
|
},
|
|
"tnews/test/3448": {
|
|
"domain": "chinese",
|
|
"within_domain_index": 2,
|
|
"selection_rank": "001d2b482c8c5d11c08d0fe0d5ce8421759f1ae362e4d6ab339a98cbcf893e3d"
|
|
},
|
|
"tnews/test/1059": {
|
|
"domain": "chinese",
|
|
"within_domain_index": 3,
|
|
"selection_rank": "003c8ebfb96b25b85cd0d61695249921a80438fbed1d314631b98720f7e9d0bd"
|
|
},
|
|
"HumanEval/31": {
|
|
"domain": "code",
|
|
"within_domain_index": 0,
|
|
"selection_rank": "004f4cfa62936a4a096e53df7d0ef3926c3ee97d32cf67a3c9bd64569f580e5c"
|
|
},
|
|
"HumanEval/44": {
|
|
"domain": "code",
|
|
"within_domain_index": 1,
|
|
"selection_rank": "030cfc7acc71ac4484d25020b2f340973c6cd1e20c2bdb1b3be9d532032eaa24"
|
|
},
|
|
"HumanEval/133": {
|
|
"domain": "code",
|
|
"within_domain_index": 2,
|
|
"selection_rank": "03661faa70d0db8ba0c1d391356f9c7cdb8f50ccbeedc0911f6b2a3be0190ed3"
|
|
},
|
|
"HumanEval/23": {
|
|
"domain": "code",
|
|
"within_domain_index": 3,
|
|
"selection_rank": "03b3c7329e474bf8e9660969ed5898d561e718f12a59d2fc1ed641be54b68860"
|
|
},
|
|
"gsm8k/test/1069": {
|
|
"domain": "math",
|
|
"within_domain_index": 0,
|
|
"selection_rank": "000946b459ce33a645f403b9d781a7aefd81f5568412149f4588381fed55b1d7"
|
|
},
|
|
"gsm8k/test/1228": {
|
|
"domain": "math",
|
|
"within_domain_index": 1,
|
|
"selection_rank": "002201d4cbc0b85d338597dcb68c3ffd4d00ef12b7b44e352c5bc1911d6fa984"
|
|
},
|
|
"gsm8k/test/0144": {
|
|
"domain": "math",
|
|
"within_domain_index": 2,
|
|
"selection_rank": "004a245538bdc42cc175a178c73dfaaaebc9713e3215ffcdfb9400fb63b3a941"
|
|
},
|
|
"gsm8k/test/1251": {
|
|
"domain": "math",
|
|
"within_domain_index": 3,
|
|
"selection_rank": "0056e66a635febc5b2ace25221ed8fe25366682898fc2d4d0132b30812f59e30"
|
|
}
|
|
},
|
|
"source_condition_cells": {
|
|
"wikitext2/raw-validation/0443": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 377,
|
|
"median": 362.0,
|
|
"min": 316,
|
|
"max": 468
|
|
},
|
|
"generated_token_seed_range": 152,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 420.75,
|
|
"median": 408.0,
|
|
"min": 355,
|
|
"max": 512
|
|
},
|
|
"generated_token_seed_range": 157,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 3,
|
|
"natural_eos_rate": 0.75,
|
|
"budget_truncated": 1,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 411,
|
|
"median": 406.0,
|
|
"min": 320,
|
|
"max": 512
|
|
},
|
|
"generated_token_seed_range": 192,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 3,
|
|
"BUDGET_TRUNCATED_UNRESOLVED": 1
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 144.75,
|
|
"median": 134.5,
|
|
"min": 121,
|
|
"max": 189
|
|
},
|
|
"generated_token_seed_range": 68,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"wikitext2/raw-validation/0030": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 265,
|
|
"median": 272.5,
|
|
"min": 64,
|
|
"max": 451
|
|
},
|
|
"generated_token_seed_range": 387,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 64,
|
|
"median": 65.5,
|
|
"min": 57,
|
|
"max": 68
|
|
},
|
|
"generated_token_seed_range": 11,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 356.25,
|
|
"median": 411.5,
|
|
"min": 105,
|
|
"max": 497
|
|
},
|
|
"generated_token_seed_range": 392,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 61.75,
|
|
"median": 67.5,
|
|
"min": 41,
|
|
"max": 71
|
|
},
|
|
"generated_token_seed_range": 30,
|
|
"unique_trajectory_hashes": 3,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"wikitext2/raw-validation/2909": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 220.5,
|
|
"median": 218.0,
|
|
"min": 147,
|
|
"max": 299
|
|
},
|
|
"generated_token_seed_range": 152,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 189,
|
|
"median": 194.5,
|
|
"min": 108,
|
|
"max": 259
|
|
},
|
|
"generated_token_seed_range": 151,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 277.5,
|
|
"median": 279.0,
|
|
"min": 249,
|
|
"max": 303
|
|
},
|
|
"generated_token_seed_range": 54,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 214,
|
|
"median": 228.0,
|
|
"min": 115,
|
|
"max": 285
|
|
},
|
|
"generated_token_seed_range": 170,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"wikitext2/raw-validation/2746": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 3,
|
|
"natural_eos_rate": 0.75,
|
|
"budget_truncated": 1,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 404.75,
|
|
"median": 415.0,
|
|
"min": 277,
|
|
"max": 512
|
|
},
|
|
"generated_token_seed_range": 235,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"BUDGET_TRUNCATED_UNRESOLVED": 1,
|
|
"NATURAL_EOS": 3
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 325.25,
|
|
"median": 323.5,
|
|
"min": 259,
|
|
"max": 395
|
|
},
|
|
"generated_token_seed_range": 136,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 2,
|
|
"natural_eos_rate": 0.5,
|
|
"budget_truncated": 2,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 468,
|
|
"median": 473.5,
|
|
"min": 413,
|
|
"max": 512
|
|
},
|
|
"generated_token_seed_range": 99,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"BUDGET_TRUNCATED_UNRESOLVED": 2,
|
|
"NATURAL_EOS": 2
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 264.75,
|
|
"median": 256.0,
|
|
"min": 246,
|
|
"max": 301
|
|
},
|
|
"generated_token_seed_range": 55,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"tnews/test/4855": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 94.25,
|
|
"median": 94.5,
|
|
"min": 69,
|
|
"max": 119
|
|
},
|
|
"generated_token_seed_range": 50,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 96,
|
|
"median": 93.0,
|
|
"min": 77,
|
|
"max": 121
|
|
},
|
|
"generated_token_seed_range": 44,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 101.75,
|
|
"median": 106.0,
|
|
"min": 75,
|
|
"max": 120
|
|
},
|
|
"generated_token_seed_range": 45,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 89.25,
|
|
"median": 87.5,
|
|
"min": 77,
|
|
"max": 105
|
|
},
|
|
"generated_token_seed_range": 28,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"tnews/test/8935": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 224.25,
|
|
"median": 247.5,
|
|
"min": 129,
|
|
"max": 273
|
|
},
|
|
"generated_token_seed_range": 144,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 262.75,
|
|
"median": 254.0,
|
|
"min": 90,
|
|
"max": 453
|
|
},
|
|
"generated_token_seed_range": 363,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 200.75,
|
|
"median": 208.5,
|
|
"min": 98,
|
|
"max": 288
|
|
},
|
|
"generated_token_seed_range": 190,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 226.25,
|
|
"median": 218.0,
|
|
"min": 207,
|
|
"max": 262
|
|
},
|
|
"generated_token_seed_range": 55,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"tnews/test/3448": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 119.25,
|
|
"median": 124.5,
|
|
"min": 87,
|
|
"max": 141
|
|
},
|
|
"generated_token_seed_range": 54,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 128.25,
|
|
"median": 135.5,
|
|
"min": 92,
|
|
"max": 150
|
|
},
|
|
"generated_token_seed_range": 58,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 98,
|
|
"median": 98.0,
|
|
"min": 91,
|
|
"max": 105
|
|
},
|
|
"generated_token_seed_range": 14,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 134.5,
|
|
"median": 87.5,
|
|
"min": 87,
|
|
"max": 276
|
|
},
|
|
"generated_token_seed_range": 189,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"tnews/test/1059": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 193.5,
|
|
"median": 196.0,
|
|
"min": 149,
|
|
"max": 233
|
|
},
|
|
"generated_token_seed_range": 84,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 193.25,
|
|
"median": 216.5,
|
|
"min": 95,
|
|
"max": 245
|
|
},
|
|
"generated_token_seed_range": 150,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 251.75,
|
|
"median": 244.5,
|
|
"min": 235,
|
|
"max": 283
|
|
},
|
|
"generated_token_seed_range": 48,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 98,
|
|
"median": 100.5,
|
|
"min": 75,
|
|
"max": 116
|
|
},
|
|
"generated_token_seed_range": 41,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": null,
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success": null,
|
|
"strict_complete_success_rate": null,
|
|
"observed_any_fixed_budget_pass": null,
|
|
"observed_any_strict_complete_pass": null,
|
|
"seed_success_range": null
|
|
}
|
|
},
|
|
"HumanEval/31": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 451,
|
|
"median": 467.5,
|
|
"min": 379,
|
|
"max": 490
|
|
},
|
|
"generated_token_seed_range": 111,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 385.25,
|
|
"median": 379.0,
|
|
"min": 350,
|
|
"max": 433
|
|
},
|
|
"generated_token_seed_range": 83,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 420,
|
|
"median": 419.5,
|
|
"min": 382,
|
|
"max": 459
|
|
},
|
|
"generated_token_seed_range": 77,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 59,
|
|
"median": 59.0,
|
|
"min": 59,
|
|
"max": 59
|
|
},
|
|
"generated_token_seed_range": 0,
|
|
"unique_trajectory_hashes": 1,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"HumanEval/44": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 474.5,
|
|
"median": 476.5,
|
|
"min": 437,
|
|
"max": 508
|
|
},
|
|
"generated_token_seed_range": 71,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 2,
|
|
"fixed_budget_success_rate": 0.5,
|
|
"strict_complete_success": 2,
|
|
"strict_complete_success_rate": 0.5,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"runtime_error": 1,
|
|
"assertion_failed": 1,
|
|
"passed": 2
|
|
}
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 3,
|
|
"natural_eos_rate": 0.75,
|
|
"budget_truncated": 1,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 433.5,
|
|
"median": 425.0,
|
|
"min": 372,
|
|
"max": 512
|
|
},
|
|
"generated_token_seed_range": 140,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 3,
|
|
"TASK_TERMINAL_BEFORE_EOS": 1
|
|
},
|
|
"fixed_budget_success": 1,
|
|
"fixed_budget_success_rate": 0.25,
|
|
"strict_complete_success": 1,
|
|
"strict_complete_success_rate": 0.25,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"runtime_error": 3,
|
|
"passed": 1
|
|
}
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 3,
|
|
"natural_eos_rate": 0.75,
|
|
"budget_truncated": 1,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 432.5,
|
|
"median": 476.0,
|
|
"min": 266,
|
|
"max": 512
|
|
},
|
|
"generated_token_seed_range": 246,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"BUDGET_TRUNCATED_UNRESOLVED": 1,
|
|
"NATURAL_EOS": 3
|
|
},
|
|
"fixed_budget_success": 1,
|
|
"fixed_budget_success_rate": 0.25,
|
|
"strict_complete_success": 1,
|
|
"strict_complete_success_rate": 0.25,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"assertion_failed": 2,
|
|
"runtime_error": 1,
|
|
"passed": 1
|
|
}
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 146.75,
|
|
"median": 164.0,
|
|
"min": 79,
|
|
"max": 180
|
|
},
|
|
"generated_token_seed_range": 101,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"HumanEval/133": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 386.75,
|
|
"median": 388.5,
|
|
"min": 366,
|
|
"max": 404
|
|
},
|
|
"generated_token_seed_range": 38,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 362.75,
|
|
"median": 360.0,
|
|
"min": 355,
|
|
"max": 376
|
|
},
|
|
"generated_token_seed_range": 21,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 343.25,
|
|
"median": 341.0,
|
|
"min": 329,
|
|
"max": 362
|
|
},
|
|
"generated_token_seed_range": 33,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 70.5,
|
|
"median": 76.0,
|
|
"min": 52,
|
|
"max": 78
|
|
},
|
|
"generated_token_seed_range": 26,
|
|
"unique_trajectory_hashes": 3,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 0,
|
|
"fixed_budget_success_rate": 0,
|
|
"strict_complete_success": 0,
|
|
"strict_complete_success_rate": 0,
|
|
"observed_any_fixed_budget_pass": false,
|
|
"observed_any_strict_complete_pass": false,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"assertion_failed": 4
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"HumanEval/23": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 107.75,
|
|
"median": 100.0,
|
|
"min": 90,
|
|
"max": 141
|
|
},
|
|
"generated_token_seed_range": 51,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 111.25,
|
|
"median": 114.0,
|
|
"min": 53,
|
|
"max": 164
|
|
},
|
|
"generated_token_seed_range": 111,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 150.5,
|
|
"median": 147.5,
|
|
"min": 116,
|
|
"max": 191
|
|
},
|
|
"generated_token_seed_range": 75,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 50.5,
|
|
"median": 50.5,
|
|
"min": 48,
|
|
"max": 53
|
|
},
|
|
"generated_token_seed_range": 5,
|
|
"unique_trajectory_hashes": 2,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"code": {
|
|
"ast_parse": 4,
|
|
"executed": 4,
|
|
"execution_statuses": {
|
|
"passed": 4
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"gsm8k/test/1069": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 188.25,
|
|
"median": 184.0,
|
|
"min": 172,
|
|
"max": 213
|
|
},
|
|
"generated_token_seed_range": 41,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 3,
|
|
"fixed_budget_success_rate": 0.75,
|
|
"strict_complete_success": 3,
|
|
"strict_complete_success_rate": 0.75,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"300": 3,
|
|
"100": 1
|
|
},
|
|
"modes": [
|
|
"300"
|
|
],
|
|
"unique_absolute_majority": "300",
|
|
"gold": "300"
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 193.5,
|
|
"median": 185.5,
|
|
"min": 175,
|
|
"max": 228
|
|
},
|
|
"generated_token_seed_range": 53,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"300": 4
|
|
},
|
|
"modes": [
|
|
"300"
|
|
],
|
|
"unique_absolute_majority": "300",
|
|
"gold": "300"
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 219,
|
|
"median": 218.0,
|
|
"min": 186,
|
|
"max": 254
|
|
},
|
|
"generated_token_seed_range": 68,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 3,
|
|
"fixed_budget_success_rate": 0.75,
|
|
"strict_complete_success": 3,
|
|
"strict_complete_success_rate": 0.75,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"100": 1,
|
|
"300": 3
|
|
},
|
|
"modes": [
|
|
"300"
|
|
],
|
|
"unique_absolute_majority": "300",
|
|
"gold": "300"
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 153.5,
|
|
"median": 149.5,
|
|
"min": 141,
|
|
"max": 174
|
|
},
|
|
"generated_token_seed_range": 33,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"300": 4
|
|
},
|
|
"modes": [
|
|
"300"
|
|
],
|
|
"unique_absolute_majority": "300",
|
|
"gold": "300"
|
|
}
|
|
}
|
|
},
|
|
"gsm8k/test/1228": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 229,
|
|
"median": 225.5,
|
|
"min": 215,
|
|
"max": 250
|
|
},
|
|
"generated_token_seed_range": 35,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"31": 4
|
|
},
|
|
"modes": [
|
|
"31"
|
|
],
|
|
"unique_absolute_majority": "31",
|
|
"gold": "31"
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 253.25,
|
|
"median": 255.5,
|
|
"min": 243,
|
|
"max": 259
|
|
},
|
|
"generated_token_seed_range": 16,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"31": 4
|
|
},
|
|
"modes": [
|
|
"31"
|
|
],
|
|
"unique_absolute_majority": "31",
|
|
"gold": "31"
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 151.25,
|
|
"median": 135.0,
|
|
"min": 72,
|
|
"max": 263
|
|
},
|
|
"generated_token_seed_range": 191,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"31": 4
|
|
},
|
|
"modes": [
|
|
"31"
|
|
],
|
|
"unique_absolute_majority": "31",
|
|
"gold": "31"
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 193.5,
|
|
"median": 223.0,
|
|
"min": 72,
|
|
"max": 256
|
|
},
|
|
"generated_token_seed_range": 184,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"31": 4
|
|
},
|
|
"modes": [
|
|
"31"
|
|
],
|
|
"unique_absolute_majority": "31",
|
|
"gold": "31"
|
|
}
|
|
}
|
|
},
|
|
"gsm8k/test/0144": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 228,
|
|
"median": 238.5,
|
|
"min": 190,
|
|
"max": 245
|
|
},
|
|
"generated_token_seed_range": 55,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 2,
|
|
"fixed_budget_success_rate": 0.5,
|
|
"strict_complete_success": 2,
|
|
"strict_complete_success_rate": 0.5,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"20": 2,
|
|
"36": 1,
|
|
"140": 1
|
|
},
|
|
"modes": [
|
|
"20"
|
|
],
|
|
"unique_absolute_majority": null,
|
|
"gold": "20"
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 219,
|
|
"median": 219.0,
|
|
"min": 211,
|
|
"max": 227
|
|
},
|
|
"generated_token_seed_range": 16,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 2,
|
|
"fixed_budget_success_rate": 0.5,
|
|
"strict_complete_success": 2,
|
|
"strict_complete_success_rate": 0.5,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"20": 2,
|
|
"36": 1,
|
|
"140": 1
|
|
},
|
|
"modes": [
|
|
"20"
|
|
],
|
|
"unique_absolute_majority": null,
|
|
"gold": "20"
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 249.75,
|
|
"median": 244.5,
|
|
"min": 150,
|
|
"max": 360
|
|
},
|
|
"generated_token_seed_range": 210,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 3,
|
|
"fixed_budget_success_rate": 0.75,
|
|
"strict_complete_success": 3,
|
|
"strict_complete_success_rate": 0.75,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"20": 3,
|
|
"36": 1
|
|
},
|
|
"modes": [
|
|
"20"
|
|
],
|
|
"unique_absolute_majority": "20",
|
|
"gold": "20"
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 215,
|
|
"median": 211.0,
|
|
"min": 201,
|
|
"max": 237
|
|
},
|
|
"generated_token_seed_range": 36,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 1,
|
|
"fixed_budget_success_rate": 0.25,
|
|
"strict_complete_success": 1,
|
|
"strict_complete_success_rate": 0.25,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"92": 2,
|
|
"20": 1,
|
|
"36": 1
|
|
},
|
|
"modes": [
|
|
"92"
|
|
],
|
|
"unique_absolute_majority": null,
|
|
"gold": "20"
|
|
}
|
|
}
|
|
},
|
|
"gsm8k/test/1251": {
|
|
"s0_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 272,
|
|
"median": 265.0,
|
|
"min": 253,
|
|
"max": 305
|
|
},
|
|
"generated_token_seed_range": 52,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 1,
|
|
"fixed_budget_success_rate": 0.25,
|
|
"strict_complete_success": 1,
|
|
"strict_complete_success_rate": 0.25,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"31.25": 1,
|
|
"42.5": 2,
|
|
"30": 1
|
|
},
|
|
"modes": [
|
|
"42.5"
|
|
],
|
|
"unique_absolute_majority": null,
|
|
"gold": "30"
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 199,
|
|
"median": 190.0,
|
|
"min": 169,
|
|
"max": 247
|
|
},
|
|
"generated_token_seed_range": 78,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 4,
|
|
"fixed_budget_success_rate": 1,
|
|
"strict_complete_success": 4,
|
|
"strict_complete_success_rate": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 0,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"30": 4
|
|
},
|
|
"modes": [
|
|
"30"
|
|
],
|
|
"unique_absolute_majority": "30",
|
|
"gold": "30"
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 220,
|
|
"median": 220.0,
|
|
"min": 183,
|
|
"max": 257
|
|
},
|
|
"generated_token_seed_range": 74,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 1,
|
|
"fixed_budget_success_rate": 0.25,
|
|
"strict_complete_success": 1,
|
|
"strict_complete_success_rate": 0.25,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"30": 1,
|
|
"6": 2,
|
|
"40": 1
|
|
},
|
|
"modes": [
|
|
"6"
|
|
],
|
|
"unique_absolute_majority": null,
|
|
"gold": "30"
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"outputs": 4,
|
|
"natural_eos": 4,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 222,
|
|
"median": 232.0,
|
|
"min": 182,
|
|
"max": 242
|
|
},
|
|
"generated_token_seed_range": 60,
|
|
"unique_trajectory_hashes": 4,
|
|
"completion_classes": {
|
|
"NATURAL_EOS": 4
|
|
},
|
|
"fixed_budget_success": 3,
|
|
"fixed_budget_success_rate": 0.75,
|
|
"strict_complete_success": 3,
|
|
"strict_complete_success_rate": 0.75,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"seed_success_range": 1,
|
|
"math_answers": {
|
|
"frequencies": {
|
|
"30": 3,
|
|
"6": 1
|
|
},
|
|
"modes": [
|
|
"30"
|
|
],
|
|
"unique_absolute_majority": "30",
|
|
"gold": "30"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"domain_condition_summary": {
|
|
"english": {
|
|
"s0_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 15,
|
|
"natural_eos_rate": 0.9375,
|
|
"budget_truncated": 1,
|
|
"mean_generated_tokens": 316.8125,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 0.9375,
|
|
"median": 1.0,
|
|
"min": 0.75,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 316.8125,
|
|
"median": 321.0,
|
|
"min": 220.5,
|
|
"max": 404.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 231.5,
|
|
"median": 193.5,
|
|
"min": 152,
|
|
"max": 387
|
|
},
|
|
"between_source_mean_length_range": 184.25,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 249.75,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 249.75,
|
|
"median": 257.125,
|
|
"min": 64,
|
|
"max": 420.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 113.75,
|
|
"median": 143.5,
|
|
"min": 11,
|
|
"max": 157
|
|
},
|
|
"between_source_mean_length_range": 356.75,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 13,
|
|
"natural_eos_rate": 0.8125,
|
|
"budget_truncated": 3,
|
|
"mean_generated_tokens": 378.1875,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 0.8125,
|
|
"median": 0.875,
|
|
"min": 0.5,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 378.1875,
|
|
"median": 383.625,
|
|
"min": 277.5,
|
|
"max": 468
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 184.25,
|
|
"median": 145.5,
|
|
"min": 54,
|
|
"max": 392
|
|
},
|
|
"between_source_mean_length_range": 190.5,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 171.3125,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 171.3125,
|
|
"median": 179.375,
|
|
"min": 61.75,
|
|
"max": 264.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 80.75,
|
|
"median": 61.5,
|
|
"min": 30,
|
|
"max": 170
|
|
},
|
|
"between_source_mean_length_range": 203.0,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
}
|
|
},
|
|
"chinese": {
|
|
"s0_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 157.8125,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 157.8125,
|
|
"median": 156.375,
|
|
"min": 94.25,
|
|
"max": 224.25
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 83,
|
|
"median": 69.0,
|
|
"min": 50,
|
|
"max": 144
|
|
},
|
|
"between_source_mean_length_range": 130.0,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 170.0625,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 170.0625,
|
|
"median": 160.75,
|
|
"min": 96,
|
|
"max": 262.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 153.75,
|
|
"median": 104.0,
|
|
"min": 44,
|
|
"max": 363
|
|
},
|
|
"between_source_mean_length_range": 166.75,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 163.0625,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 163.0625,
|
|
"median": 151.25,
|
|
"min": 98,
|
|
"max": 251.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 74.25,
|
|
"median": 46.5,
|
|
"min": 14,
|
|
"max": 190
|
|
},
|
|
"between_source_mean_length_range": 153.75,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 137,
|
|
"fixed_budget_success": null,
|
|
"strict_complete_success": null
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 137.0,
|
|
"median": 116.25,
|
|
"min": 89.25,
|
|
"max": 226.25
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 78.25,
|
|
"median": 48.0,
|
|
"min": 28,
|
|
"max": 189
|
|
},
|
|
"between_source_mean_length_range": 137.0,
|
|
"within_source_seed_success_range": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null
|
|
},
|
|
"between_source_success_rate_range": null
|
|
}
|
|
}
|
|
},
|
|
"code": {
|
|
"s0_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 355,
|
|
"fixed_budget_success": 14,
|
|
"strict_complete_success": 14
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 355.0,
|
|
"median": 418.875,
|
|
"min": 107.75,
|
|
"max": 474.5
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.875,
|
|
"median": 1.0,
|
|
"min": 0.5,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.875,
|
|
"median": 1.0,
|
|
"min": 0.5,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 67.75,
|
|
"median": 61.0,
|
|
"min": 38,
|
|
"max": 111
|
|
},
|
|
"between_source_mean_length_range": 366.75,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.5
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 15,
|
|
"natural_eos_rate": 0.9375,
|
|
"budget_truncated": 1,
|
|
"mean_generated_tokens": 323.1875,
|
|
"fixed_budget_success": 13,
|
|
"strict_complete_success": 13
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 0.9375,
|
|
"median": 1.0,
|
|
"min": 0.75,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 323.1875,
|
|
"median": 374.0,
|
|
"min": 111.25,
|
|
"max": 433.5
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.8125,
|
|
"median": 1.0,
|
|
"min": 0.25,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.8125,
|
|
"median": 1.0,
|
|
"min": 0.25,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 88.75,
|
|
"median": 97.0,
|
|
"min": 21,
|
|
"max": 140
|
|
},
|
|
"between_source_mean_length_range": 322.25,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.75
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 15,
|
|
"natural_eos_rate": 0.9375,
|
|
"budget_truncated": 1,
|
|
"mean_generated_tokens": 336.5625,
|
|
"fixed_budget_success": 13,
|
|
"strict_complete_success": 13
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 0.9375,
|
|
"median": 1.0,
|
|
"min": 0.75,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 336.5625,
|
|
"median": 381.625,
|
|
"min": 150.5,
|
|
"max": 432.5
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.8125,
|
|
"median": 1.0,
|
|
"min": 0.25,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.8125,
|
|
"median": 1.0,
|
|
"min": 0.25,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 107.75,
|
|
"median": 76.0,
|
|
"min": 33,
|
|
"max": 246
|
|
},
|
|
"between_source_mean_length_range": 282.0,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.75
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 81.6875,
|
|
"fixed_budget_success": 12,
|
|
"strict_complete_success": 12
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 81.6875,
|
|
"median": 64.75,
|
|
"min": 50.5,
|
|
"max": 146.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.75,
|
|
"median": 1.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.75,
|
|
"median": 1.0,
|
|
"min": 0,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 33,
|
|
"median": 15.5,
|
|
"min": 0,
|
|
"max": 101
|
|
},
|
|
"between_source_mean_length_range": 96.25,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0
|
|
},
|
|
"between_source_success_rate_range": 1
|
|
}
|
|
}
|
|
},
|
|
"math": {
|
|
"s0_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 229.3125,
|
|
"fixed_budget_success": 10,
|
|
"strict_complete_success": 10
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 229.3125,
|
|
"median": 228.5,
|
|
"min": 188.25,
|
|
"max": 272
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.625,
|
|
"median": 0.625,
|
|
"min": 0.25,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.625,
|
|
"median": 0.625,
|
|
"min": 0.25,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 45.75,
|
|
"median": 46.5,
|
|
"min": 35,
|
|
"max": 55
|
|
},
|
|
"between_source_mean_length_range": 83.75,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.75,
|
|
"median": 1.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.75
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 216.1875,
|
|
"fixed_budget_success": 14,
|
|
"strict_complete_success": 14
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 216.1875,
|
|
"median": 209.0,
|
|
"min": 193.5,
|
|
"max": 253.25
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.875,
|
|
"median": 1.0,
|
|
"min": 0.5,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.875,
|
|
"median": 1.0,
|
|
"min": 0.5,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 40.75,
|
|
"median": 34.5,
|
|
"min": 16,
|
|
"max": 78
|
|
},
|
|
"between_source_mean_length_range": 59.75,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.5
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 210,
|
|
"fixed_budget_success": 11,
|
|
"strict_complete_success": 11
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 210.0,
|
|
"median": 219.5,
|
|
"min": 151.25,
|
|
"max": 249.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.6875,
|
|
"median": 0.75,
|
|
"min": 0.25,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.6875,
|
|
"median": 0.75,
|
|
"min": 0.25,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 135.75,
|
|
"median": 132.5,
|
|
"min": 68,
|
|
"max": 210
|
|
},
|
|
"between_source_mean_length_range": 98.5,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.75,
|
|
"median": 1.0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.75
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"sources": 4,
|
|
"outputs": 16,
|
|
"micro": {
|
|
"natural_eos": 16,
|
|
"natural_eos_rate": 1,
|
|
"budget_truncated": 0,
|
|
"mean_generated_tokens": 196,
|
|
"fixed_budget_success": 12,
|
|
"strict_complete_success": 12
|
|
},
|
|
"source_macro": {
|
|
"natural_eos_rate": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1.0,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"mean_generated_tokens": {
|
|
"count": 4,
|
|
"mean": 196.0,
|
|
"median": 204.25,
|
|
"min": 153.5,
|
|
"max": 222
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.75,
|
|
"median": 0.875,
|
|
"min": 0.25,
|
|
"max": 1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"count": 4,
|
|
"mean": 0.75,
|
|
"median": 0.875,
|
|
"min": 0.25,
|
|
"max": 1
|
|
}
|
|
},
|
|
"variability": {
|
|
"within_source_seed_length_range": {
|
|
"count": 4,
|
|
"mean": 78.25,
|
|
"median": 48.0,
|
|
"min": 33,
|
|
"max": 184
|
|
},
|
|
"between_source_mean_length_range": 68.5,
|
|
"within_source_seed_success_range": {
|
|
"count": 4,
|
|
"mean": 0.5,
|
|
"median": 0.5,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"between_source_success_rate_range": 0.75
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"source_contrasts": {
|
|
"wikitext2/raw-validation/0443": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.125,
|
|
"boundary_main_period_minus_eos": -0.125,
|
|
"interaction": 0.25,
|
|
"period_minus_eos_s0": -0.25,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0.25
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -111.25,
|
|
"boundary_main_period_minus_eos": -121.0,
|
|
"interaction": -310.0,
|
|
"period_minus_eos_s0": 34,
|
|
"period_minus_eos_s1": -276.0,
|
|
"system_on_minus_off_eos": 43.75,
|
|
"system_on_minus_off_period": -266.25
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"wikitext2/raw-validation/0030": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -247.75,
|
|
"boundary_main_period_minus_eos": 44.5,
|
|
"interaction": -93.5,
|
|
"period_minus_eos_s0": 91.25,
|
|
"period_minus_eos_s1": -2.25,
|
|
"system_on_minus_off_eos": -201,
|
|
"system_on_minus_off_period": -294.5
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"wikitext2/raw-validation/2909": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -47.5,
|
|
"boundary_main_period_minus_eos": 41.0,
|
|
"interaction": -32.0,
|
|
"period_minus_eos_s0": 57.0,
|
|
"period_minus_eos_s1": 25,
|
|
"system_on_minus_off_eos": -31.5,
|
|
"system_on_minus_off_period": -63.5
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"wikitext2/raw-validation/2746": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.375,
|
|
"boundary_main_period_minus_eos": -0.125,
|
|
"interaction": 0.25,
|
|
"period_minus_eos_s0": -0.25,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0.25,
|
|
"system_on_minus_off_period": 0.5
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -141.375,
|
|
"boundary_main_period_minus_eos": 1.375,
|
|
"interaction": -123.75,
|
|
"period_minus_eos_s0": 63.25,
|
|
"period_minus_eos_s1": -60.5,
|
|
"system_on_minus_off_eos": -79.5,
|
|
"system_on_minus_off_period": -203.25
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"tnews/test/4855": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -5.375,
|
|
"boundary_main_period_minus_eos": 0.375,
|
|
"interaction": -14.25,
|
|
"period_minus_eos_s0": 7.5,
|
|
"period_minus_eos_s1": -6.75,
|
|
"system_on_minus_off_eos": 1.75,
|
|
"system_on_minus_off_period": -12.5
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"tnews/test/8935": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": 32.0,
|
|
"boundary_main_period_minus_eos": -30.0,
|
|
"interaction": -13.0,
|
|
"period_minus_eos_s0": -23.5,
|
|
"period_minus_eos_s1": -36.5,
|
|
"system_on_minus_off_eos": 38.5,
|
|
"system_on_minus_off_period": 25.5
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"tnews/test/3448": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": 22.75,
|
|
"boundary_main_period_minus_eos": -7.5,
|
|
"interaction": 27.5,
|
|
"period_minus_eos_s0": -21.25,
|
|
"period_minus_eos_s1": 6.25,
|
|
"system_on_minus_off_eos": 9.0,
|
|
"system_on_minus_off_period": 36.5
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"tnews/test/1059": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -77.0,
|
|
"boundary_main_period_minus_eos": -18.5,
|
|
"interaction": -153.5,
|
|
"period_minus_eos_s0": 58.25,
|
|
"period_minus_eos_s1": -95.25,
|
|
"system_on_minus_off_eos": -0.25,
|
|
"system_on_minus_off_period": -153.75
|
|
},
|
|
"fixed_budget_success_rate": null,
|
|
"strict_complete_success_rate": null
|
|
},
|
|
"HumanEval/31": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -213.375,
|
|
"boundary_main_period_minus_eos": -178.625,
|
|
"interaction": -295.25,
|
|
"period_minus_eos_s0": -31,
|
|
"period_minus_eos_s1": -326.25,
|
|
"system_on_minus_off_eos": -65.75,
|
|
"system_on_minus_off_period": -361
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
}
|
|
},
|
|
"HumanEval/44": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0.5,
|
|
"period_minus_eos_s0": -0.25,
|
|
"period_minus_eos_s1": 0.25,
|
|
"system_on_minus_off_eos": -0.25,
|
|
"system_on_minus_off_period": 0.25
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -163.375,
|
|
"boundary_main_period_minus_eos": -164.375,
|
|
"interaction": -244.75,
|
|
"period_minus_eos_s0": -42.0,
|
|
"period_minus_eos_s1": -286.75,
|
|
"system_on_minus_off_eos": -41.0,
|
|
"system_on_minus_off_period": -285.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": 0.25,
|
|
"boundary_main_period_minus_eos": 0.25,
|
|
"interaction": 1.0,
|
|
"period_minus_eos_s0": -0.25,
|
|
"period_minus_eos_s1": 0.75,
|
|
"system_on_minus_off_eos": -0.25,
|
|
"system_on_minus_off_period": 0.75
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": 0.25,
|
|
"boundary_main_period_minus_eos": 0.25,
|
|
"interaction": 1.0,
|
|
"period_minus_eos_s0": -0.25,
|
|
"period_minus_eos_s1": 0.75,
|
|
"system_on_minus_off_eos": -0.25,
|
|
"system_on_minus_off_period": 0.75
|
|
}
|
|
},
|
|
"HumanEval/133": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -148.375,
|
|
"boundary_main_period_minus_eos": -167.875,
|
|
"interaction": -248.75,
|
|
"period_minus_eos_s0": -43.5,
|
|
"period_minus_eos_s1": -292.25,
|
|
"system_on_minus_off_eos": -24.0,
|
|
"system_on_minus_off_period": -272.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": -0.5,
|
|
"boundary_main_period_minus_eos": -0.5,
|
|
"interaction": -1,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": -1,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": -1
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": -0.5,
|
|
"boundary_main_period_minus_eos": -0.5,
|
|
"interaction": -1,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": -1,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": -1
|
|
}
|
|
},
|
|
"HumanEval/23": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -48.25,
|
|
"boundary_main_period_minus_eos": -9.0,
|
|
"interaction": -103.5,
|
|
"period_minus_eos_s0": 42.75,
|
|
"period_minus_eos_s1": -60.75,
|
|
"system_on_minus_off_eos": 3.5,
|
|
"system_on_minus_off_period": -100.0
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
}
|
|
},
|
|
"gsm8k/test/1069": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -30.125,
|
|
"boundary_main_period_minus_eos": -4.625,
|
|
"interaction": -70.75,
|
|
"period_minus_eos_s0": 30.75,
|
|
"period_minus_eos_s1": -40.0,
|
|
"system_on_minus_off_eos": 5.25,
|
|
"system_on_minus_off_period": -65.5
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": 0.25,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0.0,
|
|
"period_minus_eos_s0": 0.0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0.25,
|
|
"system_on_minus_off_period": 0.25
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": 0.25,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0.0,
|
|
"period_minus_eos_s0": 0.0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0.25,
|
|
"system_on_minus_off_period": 0.25
|
|
}
|
|
},
|
|
"gsm8k/test/1228": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": 33.25,
|
|
"boundary_main_period_minus_eos": -68.75,
|
|
"interaction": 18.0,
|
|
"period_minus_eos_s0": -77.75,
|
|
"period_minus_eos_s1": -59.75,
|
|
"system_on_minus_off_eos": 24.25,
|
|
"system_on_minus_off_period": 42.25
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
}
|
|
},
|
|
"gsm8k/test/0144": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -21.875,
|
|
"boundary_main_period_minus_eos": 8.875,
|
|
"interaction": -25.75,
|
|
"period_minus_eos_s0": 21.75,
|
|
"period_minus_eos_s1": -4,
|
|
"system_on_minus_off_eos": -9,
|
|
"system_on_minus_off_period": -34.75
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": -0.25,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": -0.5,
|
|
"period_minus_eos_s0": 0.25,
|
|
"period_minus_eos_s1": -0.25,
|
|
"system_on_minus_off_eos": 0.0,
|
|
"system_on_minus_off_period": -0.5
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": -0.25,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": -0.5,
|
|
"period_minus_eos_s0": 0.25,
|
|
"period_minus_eos_s1": -0.25,
|
|
"system_on_minus_off_eos": 0.0,
|
|
"system_on_minus_off_period": -0.5
|
|
}
|
|
},
|
|
"gsm8k/test/1251": {
|
|
"natural_eos_rate": {
|
|
"system_main": 0.0,
|
|
"boundary_main_period_minus_eos": 0.0,
|
|
"interaction": 0,
|
|
"period_minus_eos_s0": 0,
|
|
"period_minus_eos_s1": 0,
|
|
"system_on_minus_off_eos": 0,
|
|
"system_on_minus_off_period": 0
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": -35.5,
|
|
"boundary_main_period_minus_eos": -14.5,
|
|
"interaction": 75,
|
|
"period_minus_eos_s0": -52,
|
|
"period_minus_eos_s1": 23,
|
|
"system_on_minus_off_eos": -73,
|
|
"system_on_minus_off_period": 2
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": 0.625,
|
|
"boundary_main_period_minus_eos": -0.125,
|
|
"interaction": -0.25,
|
|
"period_minus_eos_s0": 0.0,
|
|
"period_minus_eos_s1": -0.25,
|
|
"system_on_minus_off_eos": 0.75,
|
|
"system_on_minus_off_period": 0.5
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": 0.625,
|
|
"boundary_main_period_minus_eos": -0.125,
|
|
"interaction": -0.25,
|
|
"period_minus_eos_s0": 0.0,
|
|
"period_minus_eos_s1": -0.25,
|
|
"system_on_minus_off_eos": 0.75,
|
|
"system_on_minus_off_period": 0.5
|
|
}
|
|
}
|
|
},
|
|
"domain_contrasts": {
|
|
"english": {
|
|
"natural_eos_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": 0.125,
|
|
"median": 0.0625,
|
|
"min": 0.0,
|
|
"max": 0.375,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 2,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 0.125,
|
|
"wikitext2/raw-validation/0030": 0.0,
|
|
"wikitext2/raw-validation/2909": 0.0,
|
|
"wikitext2/raw-validation/2746": 0.375
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": -0.0625,
|
|
"min": -0.125,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 2,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -0.125,
|
|
"wikitext2/raw-validation/0030": 0.0,
|
|
"wikitext2/raw-validation/2909": 0.0,
|
|
"wikitext2/raw-validation/2746": -0.125
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": 0.125,
|
|
"median": 0.125,
|
|
"min": 0,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 2,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 0.25,
|
|
"wikitext2/raw-validation/0030": 0,
|
|
"wikitext2/raw-validation/2909": 0,
|
|
"wikitext2/raw-validation/2746": 0.25
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": -0.125,
|
|
"median": -0.125,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 2,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -0.25,
|
|
"wikitext2/raw-validation/0030": 0,
|
|
"wikitext2/raw-validation/2909": 0,
|
|
"wikitext2/raw-validation/2746": -0.25
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 0,
|
|
"wikitext2/raw-validation/0030": 0,
|
|
"wikitext2/raw-validation/2909": 0,
|
|
"wikitext2/raw-validation/2746": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 3,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 0,
|
|
"wikitext2/raw-validation/0030": 0,
|
|
"wikitext2/raw-validation/2909": 0,
|
|
"wikitext2/raw-validation/2746": 0.25
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": 0.1875,
|
|
"median": 0.125,
|
|
"min": 0,
|
|
"max": 0.5,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 2,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 0.25,
|
|
"wikitext2/raw-validation/0030": 0,
|
|
"wikitext2/raw-validation/2909": 0,
|
|
"wikitext2/raw-validation/2746": 0.5
|
|
}
|
|
}
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": -136.96875,
|
|
"median": -126.3125,
|
|
"min": -247.75,
|
|
"max": -47.5,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -111.25,
|
|
"wikitext2/raw-validation/0030": -247.75,
|
|
"wikitext2/raw-validation/2909": -47.5,
|
|
"wikitext2/raw-validation/2746": -141.375
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -8.53125,
|
|
"median": 21.1875,
|
|
"min": -121.0,
|
|
"max": 44.5,
|
|
"directions": {
|
|
"positive": 3,
|
|
"zero": 0,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -121.0,
|
|
"wikitext2/raw-validation/0030": 44.5,
|
|
"wikitext2/raw-validation/2909": 41.0,
|
|
"wikitext2/raw-validation/2746": 1.375
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": -139.8125,
|
|
"median": -108.625,
|
|
"min": -310.0,
|
|
"max": -32.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -310.0,
|
|
"wikitext2/raw-validation/0030": -93.5,
|
|
"wikitext2/raw-validation/2909": -32.0,
|
|
"wikitext2/raw-validation/2746": -123.75
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": 61.375,
|
|
"median": 60.125,
|
|
"min": 34,
|
|
"max": 91.25,
|
|
"directions": {
|
|
"positive": 4,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 34,
|
|
"wikitext2/raw-validation/0030": 91.25,
|
|
"wikitext2/raw-validation/2909": 57.0,
|
|
"wikitext2/raw-validation/2746": 63.25
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -78.4375,
|
|
"median": -31.375,
|
|
"min": -276.0,
|
|
"max": 25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -276.0,
|
|
"wikitext2/raw-validation/0030": -2.25,
|
|
"wikitext2/raw-validation/2909": 25,
|
|
"wikitext2/raw-validation/2746": -60.5
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": -67.0625,
|
|
"median": -55.5,
|
|
"min": -201,
|
|
"max": 43.75,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": 43.75,
|
|
"wikitext2/raw-validation/0030": -201,
|
|
"wikitext2/raw-validation/2909": -31.5,
|
|
"wikitext2/raw-validation/2746": -79.5
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": -206.875,
|
|
"median": -234.75,
|
|
"min": -294.5,
|
|
"max": -63.5,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": -266.25,
|
|
"wikitext2/raw-validation/0030": -294.5,
|
|
"wikitext2/raw-validation/2909": -63.5,
|
|
"wikitext2/raw-validation/2746": -203.25
|
|
}
|
|
}
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"interaction": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
}
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"interaction": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
}
|
|
}
|
|
},
|
|
"chinese": {
|
|
"natural_eos_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0.0,
|
|
"tnews/test/8935": 0.0,
|
|
"tnews/test/3448": 0.0,
|
|
"tnews/test/1059": 0.0
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0.0,
|
|
"tnews/test/8935": 0.0,
|
|
"tnews/test/3448": 0.0,
|
|
"tnews/test/1059": 0.0
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0,
|
|
"tnews/test/8935": 0,
|
|
"tnews/test/3448": 0,
|
|
"tnews/test/1059": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0,
|
|
"tnews/test/8935": 0,
|
|
"tnews/test/3448": 0,
|
|
"tnews/test/1059": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0,
|
|
"tnews/test/8935": 0,
|
|
"tnews/test/3448": 0,
|
|
"tnews/test/1059": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0,
|
|
"tnews/test/8935": 0,
|
|
"tnews/test/3448": 0,
|
|
"tnews/test/1059": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0,
|
|
"tnews/test/8935": 0,
|
|
"tnews/test/3448": 0,
|
|
"tnews/test/1059": 0
|
|
}
|
|
}
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": -6.90625,
|
|
"median": 8.6875,
|
|
"min": -77.0,
|
|
"max": 32.0,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": -5.375,
|
|
"tnews/test/8935": 32.0,
|
|
"tnews/test/3448": 22.75,
|
|
"tnews/test/1059": -77.0
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -13.90625,
|
|
"median": -13.0,
|
|
"min": -30.0,
|
|
"max": 0.375,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 0.375,
|
|
"tnews/test/8935": -30.0,
|
|
"tnews/test/3448": -7.5,
|
|
"tnews/test/1059": -18.5
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": -38.3125,
|
|
"median": -13.625,
|
|
"min": -153.5,
|
|
"max": 27.5,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": -14.25,
|
|
"tnews/test/8935": -13.0,
|
|
"tnews/test/3448": 27.5,
|
|
"tnews/test/1059": -153.5
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": 5.25,
|
|
"median": -6.875,
|
|
"min": -23.5,
|
|
"max": 58.25,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 7.5,
|
|
"tnews/test/8935": -23.5,
|
|
"tnews/test/3448": -21.25,
|
|
"tnews/test/1059": 58.25
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -33.0625,
|
|
"median": -21.625,
|
|
"min": -95.25,
|
|
"max": 6.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": -6.75,
|
|
"tnews/test/8935": -36.5,
|
|
"tnews/test/3448": 6.25,
|
|
"tnews/test/1059": -95.25
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": 12.25,
|
|
"median": 5.375,
|
|
"min": -0.25,
|
|
"max": 38.5,
|
|
"directions": {
|
|
"positive": 3,
|
|
"zero": 0,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": 1.75,
|
|
"tnews/test/8935": 38.5,
|
|
"tnews/test/3448": 9.0,
|
|
"tnews/test/1059": -0.25
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": -26.0625,
|
|
"median": 6.5,
|
|
"min": -153.75,
|
|
"max": 36.5,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"tnews/test/4855": -12.5,
|
|
"tnews/test/8935": 25.5,
|
|
"tnews/test/3448": 36.5,
|
|
"tnews/test/1059": -153.75
|
|
}
|
|
}
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"interaction": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
}
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"interaction": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 0,
|
|
"mean": null,
|
|
"median": null,
|
|
"min": null,
|
|
"max": null,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 0
|
|
},
|
|
"by_source": {}
|
|
}
|
|
}
|
|
},
|
|
"code": {
|
|
"natural_eos_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0.0,
|
|
"HumanEval/44": 0.0,
|
|
"HumanEval/133": 0.0,
|
|
"HumanEval/23": 0.0
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0.0,
|
|
"HumanEval/44": 0.0,
|
|
"HumanEval/133": 0.0,
|
|
"HumanEval/23": 0.0
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": 0.125,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0.5,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 3,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.5,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": -0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 3,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": -0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 3,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
}
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": -143.34375,
|
|
"median": -155.875,
|
|
"min": -213.375,
|
|
"max": -48.25,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -213.375,
|
|
"HumanEval/44": -163.375,
|
|
"HumanEval/133": -148.375,
|
|
"HumanEval/23": -48.25
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -129.96875,
|
|
"median": -166.125,
|
|
"min": -178.625,
|
|
"max": -9.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -178.625,
|
|
"HumanEval/44": -164.375,
|
|
"HumanEval/133": -167.875,
|
|
"HumanEval/23": -9.0
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": -223.0625,
|
|
"median": -246.75,
|
|
"min": -295.25,
|
|
"max": -103.5,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -295.25,
|
|
"HumanEval/44": -244.75,
|
|
"HumanEval/133": -248.75,
|
|
"HumanEval/23": -103.5
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": -18.4375,
|
|
"median": -36.5,
|
|
"min": -43.5,
|
|
"max": 42.75,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -31,
|
|
"HumanEval/44": -42.0,
|
|
"HumanEval/133": -43.5,
|
|
"HumanEval/23": 42.75
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -241.5,
|
|
"median": -289.5,
|
|
"min": -326.25,
|
|
"max": -60.75,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -326.25,
|
|
"HumanEval/44": -286.75,
|
|
"HumanEval/133": -292.25,
|
|
"HumanEval/23": -60.75
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": -31.8125,
|
|
"median": -32.5,
|
|
"min": -65.75,
|
|
"max": 3.5,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -65.75,
|
|
"HumanEval/44": -41.0,
|
|
"HumanEval/133": -24.0,
|
|
"HumanEval/23": 3.5
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": -254.875,
|
|
"median": -279.25,
|
|
"min": -361,
|
|
"max": -100.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 0,
|
|
"negative": 4
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": -361,
|
|
"HumanEval/44": -285.75,
|
|
"HumanEval/133": -272.75,
|
|
"HumanEval/23": -100.0
|
|
}
|
|
}
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.5,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0.0,
|
|
"HumanEval/44": 0.25,
|
|
"HumanEval/133": -0.5,
|
|
"HumanEval/23": 0.0
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.5,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0.0,
|
|
"HumanEval/44": 0.25,
|
|
"HumanEval/133": -0.5,
|
|
"HumanEval/23": 0.0
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": -1,
|
|
"max": 1.0,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 1.0,
|
|
"HumanEval/133": -1,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": -0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -1,
|
|
"max": 0.75,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.75,
|
|
"HumanEval/133": -1,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": -0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -1,
|
|
"max": 0.75,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.75,
|
|
"HumanEval/133": -1,
|
|
"HumanEval/23": 0
|
|
}
|
|
}
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.5,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0.0,
|
|
"HumanEval/44": 0.25,
|
|
"HumanEval/133": -0.5,
|
|
"HumanEval/23": 0.0
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.5,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0.0,
|
|
"HumanEval/44": 0.25,
|
|
"HumanEval/133": -0.5,
|
|
"HumanEval/23": 0.0
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": -1,
|
|
"max": 1.0,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 1.0,
|
|
"HumanEval/133": -1,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": -0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -1,
|
|
"max": 0.75,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.75,
|
|
"HumanEval/133": -1,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": -0.25,
|
|
"HumanEval/133": 0,
|
|
"HumanEval/23": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": -0.0625,
|
|
"median": 0.0,
|
|
"min": -1,
|
|
"max": 0.75,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 2,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"HumanEval/31": 0,
|
|
"HumanEval/44": 0.75,
|
|
"HumanEval/133": -1,
|
|
"HumanEval/23": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"math": {
|
|
"natural_eos_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0.0,
|
|
"gsm8k/test/0144": 0.0,
|
|
"gsm8k/test/1251": 0.0
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": 0.0,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0.0,
|
|
"gsm8k/test/0144": 0.0,
|
|
"gsm8k/test/1251": 0.0
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0,
|
|
"gsm8k/test/1251": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0,
|
|
"gsm8k/test/1251": 0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0,
|
|
"gsm8k/test/1251": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0,
|
|
"gsm8k/test/1251": 0
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0.0,
|
|
"min": 0,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 4,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0,
|
|
"gsm8k/test/1251": 0
|
|
}
|
|
}
|
|
},
|
|
"mean_generated_tokens": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": -13.5625,
|
|
"median": -26.0,
|
|
"min": -35.5,
|
|
"max": 33.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": -30.125,
|
|
"gsm8k/test/1228": 33.25,
|
|
"gsm8k/test/0144": -21.875,
|
|
"gsm8k/test/1251": -35.5
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -19.75,
|
|
"median": -9.5625,
|
|
"min": -68.75,
|
|
"max": 8.875,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": -4.625,
|
|
"gsm8k/test/1228": -68.75,
|
|
"gsm8k/test/0144": 8.875,
|
|
"gsm8k/test/1251": -14.5
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": -0.875,
|
|
"median": -3.875,
|
|
"min": -70.75,
|
|
"max": 75,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": -70.75,
|
|
"gsm8k/test/1228": 18.0,
|
|
"gsm8k/test/0144": -25.75,
|
|
"gsm8k/test/1251": 75
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": -19.3125,
|
|
"median": -15.125,
|
|
"min": -77.75,
|
|
"max": 30.75,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 30.75,
|
|
"gsm8k/test/1228": -77.75,
|
|
"gsm8k/test/0144": 21.75,
|
|
"gsm8k/test/1251": -52
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -20.1875,
|
|
"median": -22.0,
|
|
"min": -59.75,
|
|
"max": 23,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 0,
|
|
"negative": 3
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": -40.0,
|
|
"gsm8k/test/1228": -59.75,
|
|
"gsm8k/test/0144": -4,
|
|
"gsm8k/test/1251": 23
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": -13.125,
|
|
"median": -1.875,
|
|
"min": -73,
|
|
"max": 24.25,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 5.25,
|
|
"gsm8k/test/1228": 24.25,
|
|
"gsm8k/test/0144": -9,
|
|
"gsm8k/test/1251": -73
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": -14.0,
|
|
"median": -16.375,
|
|
"min": -65.5,
|
|
"max": 42.25,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 0,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": -65.5,
|
|
"gsm8k/test/1228": 42.25,
|
|
"gsm8k/test/0144": -34.75,
|
|
"gsm8k/test/1251": 2
|
|
}
|
|
}
|
|
},
|
|
"fixed_budget_success_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": 0.15625,
|
|
"median": 0.125,
|
|
"min": -0.25,
|
|
"max": 0.625,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 1,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.25,
|
|
"gsm8k/test/1228": 0.0,
|
|
"gsm8k/test/0144": -0.25,
|
|
"gsm8k/test/1251": 0.625
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -0.03125,
|
|
"median": 0.0,
|
|
"min": -0.125,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0.0,
|
|
"gsm8k/test/0144": 0.0,
|
|
"gsm8k/test/1251": -0.125
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": -0.1875,
|
|
"median": -0.125,
|
|
"min": -0.5,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 2,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": -0.5,
|
|
"gsm8k/test/1251": -0.25
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 3,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0.25,
|
|
"gsm8k/test/1251": 0.0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -0.125,
|
|
"median": -0.125,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 2,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": -0.25,
|
|
"gsm8k/test/1251": -0.25
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0.125,
|
|
"min": 0,
|
|
"max": 0.75,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 2,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.25,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0.0,
|
|
"gsm8k/test/1251": 0.75
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.125,
|
|
"min": -0.5,
|
|
"max": 0.5,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 1,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.25,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": -0.5,
|
|
"gsm8k/test/1251": 0.5
|
|
}
|
|
}
|
|
},
|
|
"strict_complete_success_rate": {
|
|
"system_main": {
|
|
"count": 4,
|
|
"mean": 0.15625,
|
|
"median": 0.125,
|
|
"min": -0.25,
|
|
"max": 0.625,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 1,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.25,
|
|
"gsm8k/test/1228": 0.0,
|
|
"gsm8k/test/0144": -0.25,
|
|
"gsm8k/test/1251": 0.625
|
|
}
|
|
},
|
|
"boundary_main_period_minus_eos": {
|
|
"count": 4,
|
|
"mean": -0.03125,
|
|
"median": 0.0,
|
|
"min": -0.125,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 3,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0.0,
|
|
"gsm8k/test/0144": 0.0,
|
|
"gsm8k/test/1251": -0.125
|
|
}
|
|
},
|
|
"interaction": {
|
|
"count": 4,
|
|
"mean": -0.1875,
|
|
"median": -0.125,
|
|
"min": -0.5,
|
|
"max": 0.0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 2,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": -0.5,
|
|
"gsm8k/test/1251": -0.25
|
|
}
|
|
},
|
|
"period_minus_eos_s0": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.0,
|
|
"min": 0.0,
|
|
"max": 0.25,
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 3,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0.25,
|
|
"gsm8k/test/1251": 0.0
|
|
}
|
|
},
|
|
"period_minus_eos_s1": {
|
|
"count": 4,
|
|
"mean": -0.125,
|
|
"median": -0.125,
|
|
"min": -0.25,
|
|
"max": 0,
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 2,
|
|
"negative": 2
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": -0.25,
|
|
"gsm8k/test/1251": -0.25
|
|
}
|
|
},
|
|
"system_on_minus_off_eos": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0.125,
|
|
"min": 0,
|
|
"max": 0.75,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 2,
|
|
"negative": 0
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.25,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": 0.0,
|
|
"gsm8k/test/1251": 0.75
|
|
}
|
|
},
|
|
"system_on_minus_off_period": {
|
|
"count": 4,
|
|
"mean": 0.0625,
|
|
"median": 0.125,
|
|
"min": -0.5,
|
|
"max": 0.5,
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 1,
|
|
"negative": 1
|
|
},
|
|
"by_source": {
|
|
"gsm8k/test/1069": 0.25,
|
|
"gsm8k/test/1228": 0,
|
|
"gsm8k/test/0144": -0.5,
|
|
"gsm8k/test/1251": 0.5
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"task_matrix": {
|
|
"code": [
|
|
{
|
|
"source_id": "HumanEval/31",
|
|
"within_domain_index": 0,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 451,
|
|
"median": 467.5,
|
|
"min": 379,
|
|
"max": 490
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 385.25,
|
|
"median": 379.0,
|
|
"min": 350,
|
|
"max": 433
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 420,
|
|
"median": 419.5,
|
|
"min": 382,
|
|
"max": 459
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 59,
|
|
"median": 59.0,
|
|
"min": 59,
|
|
"max": 59
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"source_id": "HumanEval/44",
|
|
"within_domain_index": 1,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 2,
|
|
"strict_complete_success": 2,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 474.5,
|
|
"median": 476.5,
|
|
"min": 437,
|
|
"max": 508
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 1,
|
|
"strict_complete_success": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 3,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 433.5,
|
|
"median": 425.0,
|
|
"min": 372,
|
|
"max": 512
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 1,
|
|
"strict_complete_success": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 3,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 432.5,
|
|
"median": 476.0,
|
|
"min": 266,
|
|
"max": 512
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 146.75,
|
|
"median": 164.0,
|
|
"min": 79,
|
|
"max": 180
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"source_id": "HumanEval/133",
|
|
"within_domain_index": 2,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 386.75,
|
|
"median": 388.5,
|
|
"min": 366,
|
|
"max": 404
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 362.75,
|
|
"median": 360.0,
|
|
"min": 355,
|
|
"max": 376
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 343.25,
|
|
"median": 341.0,
|
|
"min": 329,
|
|
"max": 362
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 0,
|
|
"strict_complete_success": 0,
|
|
"observed_any_fixed_budget_pass": false,
|
|
"observed_any_strict_complete_pass": false,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 70.5,
|
|
"median": 76.0,
|
|
"min": 52,
|
|
"max": 78
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"source_id": "HumanEval/23",
|
|
"within_domain_index": 3,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 107.75,
|
|
"median": 100.0,
|
|
"min": 90,
|
|
"max": 141
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 111.25,
|
|
"median": 114.0,
|
|
"min": 53,
|
|
"max": 164
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 150.5,
|
|
"median": 147.5,
|
|
"min": 116,
|
|
"max": 191
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 50.5,
|
|
"median": 50.5,
|
|
"min": 48,
|
|
"max": 53
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"math": [
|
|
{
|
|
"source_id": "gsm8k/test/1069",
|
|
"within_domain_index": 0,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 3,
|
|
"strict_complete_success": 3,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 188.25,
|
|
"median": 184.0,
|
|
"min": 172,
|
|
"max": 213
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 193.5,
|
|
"median": 185.5,
|
|
"min": 175,
|
|
"max": 228
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 3,
|
|
"strict_complete_success": 3,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 219,
|
|
"median": 218.0,
|
|
"min": 186,
|
|
"max": 254
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 153.5,
|
|
"median": 149.5,
|
|
"min": 141,
|
|
"max": 174
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"source_id": "gsm8k/test/1228",
|
|
"within_domain_index": 1,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 229,
|
|
"median": 225.5,
|
|
"min": 215,
|
|
"max": 250
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 253.25,
|
|
"median": 255.5,
|
|
"min": 243,
|
|
"max": 259
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 151.25,
|
|
"median": 135.0,
|
|
"min": 72,
|
|
"max": 263
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 193.5,
|
|
"median": 223.0,
|
|
"min": 72,
|
|
"max": 256
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"source_id": "gsm8k/test/0144",
|
|
"within_domain_index": 2,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 2,
|
|
"strict_complete_success": 2,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 228,
|
|
"median": 238.5,
|
|
"min": 190,
|
|
"max": 245
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 2,
|
|
"strict_complete_success": 2,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 219,
|
|
"median": 219.0,
|
|
"min": 211,
|
|
"max": 227
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 3,
|
|
"strict_complete_success": 3,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 249.75,
|
|
"median": 244.5,
|
|
"min": 150,
|
|
"max": 360
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 1,
|
|
"strict_complete_success": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 215,
|
|
"median": 211.0,
|
|
"min": 201,
|
|
"max": 237
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"source_id": "gsm8k/test/1251",
|
|
"within_domain_index": 3,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"fixed_budget_success": 1,
|
|
"strict_complete_success": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 272,
|
|
"median": 265.0,
|
|
"min": 253,
|
|
"max": 305
|
|
}
|
|
},
|
|
"s1_eos": {
|
|
"fixed_budget_success": 4,
|
|
"strict_complete_success": 4,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 199,
|
|
"median": 190.0,
|
|
"min": 169,
|
|
"max": 247
|
|
}
|
|
},
|
|
"s0_period": {
|
|
"fixed_budget_success": 1,
|
|
"strict_complete_success": 1,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 220,
|
|
"median": 220.0,
|
|
"min": 183,
|
|
"max": 257
|
|
}
|
|
},
|
|
"s1_period": {
|
|
"fixed_budget_success": 3,
|
|
"strict_complete_success": 3,
|
|
"observed_any_fixed_budget_pass": true,
|
|
"observed_any_strict_complete_pass": true,
|
|
"natural_eos": 4,
|
|
"generated_tokens": {
|
|
"count": 4,
|
|
"mean": 222,
|
|
"median": 232.0,
|
|
"min": 182,
|
|
"max": 242
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"prior_direction_check": {
|
|
"english": {
|
|
"period_shortens_mean_tokens": {
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": true,
|
|
"wikitext2/raw-validation/0030": false,
|
|
"wikitext2/raw-validation/2909": false,
|
|
"wikitext2/raw-validation/2746": false
|
|
}
|
|
},
|
|
"system_on_raises_natural_eos_rate": {
|
|
"by_source": {
|
|
"wikitext2/raw-validation/0443": true,
|
|
"wikitext2/raw-validation/0030": false,
|
|
"wikitext2/raw-validation/2909": false,
|
|
"wikitext2/raw-validation/2746": true
|
|
}
|
|
}
|
|
},
|
|
"chinese": {
|
|
"period_shortens_mean_tokens": {
|
|
"by_source": {
|
|
"tnews/test/4855": false,
|
|
"tnews/test/8935": true,
|
|
"tnews/test/3448": true,
|
|
"tnews/test/1059": true
|
|
}
|
|
},
|
|
"system_on_raises_natural_eos_rate": {
|
|
"by_source": {
|
|
"tnews/test/4855": false,
|
|
"tnews/test/8935": false,
|
|
"tnews/test/3448": false,
|
|
"tnews/test/1059": false
|
|
}
|
|
}
|
|
},
|
|
"code": {
|
|
"period_shortens_mean_tokens": {
|
|
"by_source": {
|
|
"HumanEval/31": true,
|
|
"HumanEval/44": true,
|
|
"HumanEval/133": true,
|
|
"HumanEval/23": true
|
|
}
|
|
},
|
|
"system_on_raises_natural_eos_rate": {
|
|
"by_source": {
|
|
"HumanEval/31": false,
|
|
"HumanEval/44": false,
|
|
"HumanEval/133": false,
|
|
"HumanEval/23": false
|
|
}
|
|
}
|
|
},
|
|
"math": {
|
|
"period_shortens_mean_tokens": {
|
|
"by_source": {
|
|
"gsm8k/test/1069": true,
|
|
"gsm8k/test/1228": true,
|
|
"gsm8k/test/0144": false,
|
|
"gsm8k/test/1251": true
|
|
}
|
|
},
|
|
"system_on_raises_natural_eos_rate": {
|
|
"by_source": {
|
|
"gsm8k/test/1069": false,
|
|
"gsm8k/test/1228": false,
|
|
"gsm8k/test/0144": false,
|
|
"gsm8k/test/1251": false
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"claim_boundary": [
|
|
"Four sources per domain are not full benchmark estimates.",
|
|
"Seed-level repeats are not independent task observations.",
|
|
"Source-blocked contrasts are descriptive and have no p-values.",
|
|
"Observed any-pass is not standard HumanEval pass@4.",
|
|
"Natural EOS, evaluator coverage, and correctness remain separate.",
|
|
"The period condition is a counterfactual, not an official-valid chat.",
|
|
"Round 06 motivated the period contrast, so this is a directed follow-up."
|
|
],
|
|
"content_hash": "9c957e3bc468931c7d0b38ae3285fa5509015d3feb7cf9d4a61ce4c6b5c2f64b"
|
|
}
|