From 6181af90c85aed50bc286abf54890c59d7b8b49f Mon Sep 17 00:00:00 2001 From: wuyang <5700876+banisherwy@user.noreply.gitee.com> Date: Thu, 30 Jul 2026 04:14:58 +0800 Subject: [PATCH] research: preregister task bootstrap CRN protocol --- ..._V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json | 1304 +++++++++++++++++ ...EK_V2_LITE_CHAT_TASK_BOOTSTRAP_PROTOCOL.md | 381 +++++ ...-deepseek-chat-task-bootstrap-manifest.mjs | 198 +++ 3 files changed, 1883 insertions(+) create mode 100644 research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json create mode 100644 research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_PROTOCOL.md create mode 100644 scripts/build-deepseek-chat-task-bootstrap-manifest.mjs diff --git a/research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json b/research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json new file mode 100644 index 0000000..17208f8 --- /dev/null +++ b/research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json @@ -0,0 +1,1304 @@ +{ + "schema_version": 1, + "protocol_id": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1", + "status": "preregistered_before_any_protocol_output", + "source_reference": { + "path": "src/data/deepseek-v2-lite-routing-special-token-family-control.json", + "sha256": "c372c1b03a8b15f615b54ded5d9257a8fc2cdb7728001735d3d4c1d8534af5bf", + "sample_salt": "llm-atlas-deepseek-routing-template-control-v1", + "selection_rule": "For each domain, all 32 rows ordered by frozen within_domain_index; no outcome-based selection." + }, + "model": { + "repo": "deepseek-ai/DeepSeek-V2-Lite-Chat", + "revision": "85864749cd611b4353ce1decdb286193298f64c7" + }, + "conditions": [ + "s0_eos", + "s1_eos", + "s0_period", + "s1_period" + ], + "domains": [ + "code", + "math" + ], + "sources": [ + { + "id": "HumanEval/31", + "domain": "code", + "domain_index": 0, + "selection_rank": "004f4cfa62936a4a096e53df7d0ef3926c3ee97d32cf67a3c9bd64569f580e5c", + "source_text_sha256": "b7e32acf5a6b551a782eacfb4ecab09e59f0577a2da9642e292d2479af5bf50e", + "prompt_token_ids_sha256": { + "s0_eos": "7b77b4577b783e49ff43d4c2187d0351070e82d9f7ad6407d456e710997d341a", + "s1_eos": "692fe7aa84f0d62d4706eb2c07f7c87602e13a16b2ea4753b277222bd9c2cd27", + "s0_period": "421b6cfe57a7662ce1c9fecb7761f01a723fbddae6956634deef5b0c855c0ca0", + "s1_period": "a3bb871efed93a2777818807aaefca4dd9b24866ca041312b73e7a300ed356be" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "HumanEval/44", + "domain": "code", + "domain_index": 1, + "selection_rank": "030cfc7acc71ac4484d25020b2f340973c6cd1e20c2bdb1b3be9d532032eaa24", + "source_text_sha256": "70a0af759ec21f43ca0d40f7b7a3ec39e32388c9b5416fe06e5c658ce89e6666", + "prompt_token_ids_sha256": { + "s0_eos": "d307dd3753328321025f1c4f7daed99d8112cdbaece9ae5c4e9434ec0be99bea", + "s1_eos": "08438585e3cf69835d23eeb8ab6f1090d12c4f84f8b0565522bc2da770b5e044", + "s0_period": "113b960a9ba5cb68975b9525955d3c500de2a9e7dac65516d905ee47526ce4e9", + "s1_period": "98704e8f1d3d56e2632884512147201d6bb36b9f5276a9a3425004830faf762a" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/133", + "domain": "code", + "domain_index": 2, + "selection_rank": "03661faa70d0db8ba0c1d391356f9c7cdb8f50ccbeedc0911f6b2a3be0190ed3", + "source_text_sha256": "8c46f1b09d1a8179497c272a2597766eb2a53fbd00cb8cdd8a8446e547103a13", + "prompt_token_ids_sha256": { + "s0_eos": "eff8116b77e3d06fb232cfb8d6eeb50a596d65d6d1b36793dbf64ccb6e60cb89", + "s1_eos": "8a62de2d1348a57470e6b2fbf13ff05e92df2beb736f6c395645b1f725255306", + "s0_period": "de16ffc772f59282841164b0f63a3123ea9f4afa611b957da7dd22e5738e79a3", + "s1_period": "54cd325ea739b6de25749918e7d480b91e42f9712c0c228bad66f93e897a9877" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/23", + "domain": "code", + "domain_index": 3, + "selection_rank": "03b3c7329e474bf8e9660969ed5898d561e718f12a59d2fc1ed641be54b68860", + "source_text_sha256": "0bb479f80d27872f6bebc01618112b372cc8a2e6aee9e4b76337e3b3416418c7", + "prompt_token_ids_sha256": { + "s0_eos": "44f56cefedb9e06886f39ae8d3cb2dc2fbf272474cfe90f674c27903cd9260f7", + "s1_eos": "ccfd99b2ec8a496df3f757f161a19c5c12b510c89c4d022d9ea60a1369d90750", + "s0_period": "9b42bf10687a2d2bce7d8dbc9e9f7514e83f6d4eb7477db407154794f07eb39f", + "s1_period": "c3228b385245203c34c2b7c94b8a58c04ae96ee64d5d9b21ffe01bfa6a906175" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/147", + "domain": "code", + "domain_index": 4, + "selection_rank": "05f542d26e3e5561da1a47e3b05d19f187ae9cd59e49ffb7a068f6bf89ddb892", + "source_text_sha256": "d877205eaa6209a40f4f1c45755d9f04f038e64c49a366a08322a5111b05ae40", + "prompt_token_ids_sha256": { + "s0_eos": "936dda23f081877f6e706ec1d1d65a132a787352fb1f0819de02f832602389a7", + "s1_eos": "7731722269c215ebdf607875fdb77a59d3481c0de5d935191b027453ffd80bd1", + "s0_period": "444f299cb5691cbe86e7fe70e7859b7175728d0c8b64c8cf23e6c78ad62e9de6", + "s1_period": "afe8bf5937d011b2f5e3aedd6eefeb8b066557205fdc3f093bf0c11a770bb9a5" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "HumanEval/34", + "domain": "code", + "domain_index": 5, + "selection_rank": "065d54b245ce523be85bc67060106d98536bacf8e862d709bc254ba6f24cca75", + "source_text_sha256": "f3ad3f30094ecb6a48879e4b24e7e94112ca6c5e50fc82d98a4622f11d14839a", + "prompt_token_ids_sha256": { + "s0_eos": "8c004a5eceb1743d097a8b3073ea6bc4638359c63a2d254b9a2d8df4554d76f4", + "s1_eos": "59fa486fa217285d65df11f5db824be7d6680ef437379e31f6fc83acb6204ac6", + "s0_period": "7cd611f6dee5f07d3f20c500091953cadddf5768247e544d4094aa5a8bac0b0b", + "s1_period": "5cabf14197b694dd058f4fccdcf355f325b44f8ded5fbd93615d4e0d89ff0659" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/123", + "domain": "code", + "domain_index": 6, + "selection_rank": "0782f17352664fabe6a6a9e2bd72a83d60ba60aaaed7900593390e1f16d00241", + "source_text_sha256": "b74013271763bb6e1b0f3a7cd6e2f489c5205edd219a55145ba2409b52cf2c44", + "prompt_token_ids_sha256": { + "s0_eos": "153d82ed03de25c51393e3d5eb83ffdf1ff238d3c70f52c3bbb1bb21c3d26de9", + "s1_eos": "1de21d2674032ed1caf812f40917bc3ed5053244ae265a515788f92271fd194a", + "s0_period": "28564bef8d7b9c55fb0e67f4e1a17f1f509927aaf0022df66983eb0a7ea77f5e", + "s1_period": "35ecd17721d268175edbd0b3020be5e8981cd4d51fec921ad9e6e78cc00e67fa" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/77", + "domain": "code", + "domain_index": 7, + "selection_rank": "080caedf33fbd9532bd76e2b8966df2f14fcca78e937976441b602e3a4374a53", + "source_text_sha256": "44c77e3178c60f7a753765c62cf0edfdab0da4cc1b19b2ebd9cd88aad46729f5", + "prompt_token_ids_sha256": { + "s0_eos": "daee0dfff5fddd485e3c889c4decbb6915ef82862eba61ecb72df985acfd16a7", + "s1_eos": "5b426ad93113e0721440bf05460a5cb4fb99b93fb4595b9921063e908823ed67", + "s0_period": "d9d41156b1bbadd7974085db0a24319fdfef160c8b42c5e06f6d9419342720de", + "s1_period": "b8a89e7f9c57f002fb82f2214ae9a1ed5844df94f829ebb17a122937b02789f7" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/22", + "domain": "code", + "domain_index": 8, + "selection_rank": "0a405335d6bb665be461ddda5d6e1ec3a7adef8e224bc35ab268d1851def9a54", + "source_text_sha256": "45a5283612af3caab31e93add5e1b110b3616e6d61dd9c7d8f71bad625468b9d", + "prompt_token_ids_sha256": { + "s0_eos": "d259f0ca39731ed364a920f618ede0fe59daf42561d2b4cfcc978f7323c852e0", + "s1_eos": "c9f1f9e99f2760bf1ef1f940281489c4794ee218e1da67bef9a109aa51c7ec3d", + "s0_period": "cdc178e573b59f0bd3a4df10cfa635c313c95c58ef30c894ab9a8e94460f6789", + "s1_period": "b32897dfa8479dfbcc2fd34c3238465b6f1ff8d3a32021d0cb4e0c7b76a962b6" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "HumanEval/127", + "domain": "code", + "domain_index": 9, + "selection_rank": "0abbfb91f73663bb04458c0b3cac62fbe2aaf3374b7ccb9b26b1e1db67a4d4d2", + "source_text_sha256": "897803d2df27adaf9ee777c6b78f18fb771381928fbf2774e991411b15c340ad", + "prompt_token_ids_sha256": { + "s0_eos": "b462b3f52193fe601d58f7efc9efc5bdd21df100b6cc91068e0bc7599f0e6296", + "s1_eos": "2f55735917f9d5686b161b275f6482cea0da886a5b670a6d1dbb05d5ebbaec86", + "s0_period": "565f7eddd88b6e6ebaf95704022b95850b17824947f7012fab46d9a0196ccd10", + "s1_period": "2e1a7837f8b37aa905987947311f43271d2ef15defb527dfa2d881560517c35c" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/37", + "domain": "code", + "domain_index": 10, + "selection_rank": "0c9e63208d91d58824926ce0f501a49a3e55deecc293d9ebac84afc9d16182dd", + "source_text_sha256": "229eeb9f0db57f5c2dde0a63ddba58a271940904b663b063f5dd32d79d0d8bfe", + "prompt_token_ids_sha256": { + "s0_eos": "bd85acb66072986828bf7fcdcc57db2e70ca73ba5b8d73c9e72315ba69797ce5", + "s1_eos": "43b46fef3e583c653403e2c39eb60211e4c7a6fe17c99525bd2d3670e780edcf", + "s0_period": "8bd1494d3c196c05063d1d23c1806a7ba1c771dee7422c7e807a4553062e7458", + "s1_period": "bc2c6624ab4c64633edcb04f92a39fc15bd2bf131a02c4188e65e51be2f9d788" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/48", + "domain": "code", + "domain_index": 11, + "selection_rank": "0df5efc67f743da62c6c8793675ebce590c03f71d978e95d04527ced80ab6069", + "source_text_sha256": "66c7b9796a6cfafed4f62fd1e784483b287656c7a628d39582e747967349220d", + "prompt_token_ids_sha256": { + "s0_eos": "5770afc438461941f106c8736e5c241a6dc743347f6f1e9c5d3f3ee44b96d678", + "s1_eos": "76e83c1beb445d4f76d8f97697feeea7ec26ce70ba41434ce2a8ea609cbd69ba", + "s0_period": "91608b62ed4f3b86fc2077421ea9fbbcfcf18f450d5f348f062877738812f505", + "s1_period": "6b437177e75a324073848cfdbc77f603f4a8db01586c3878817b2541db802e50" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/158", + "domain": "code", + "domain_index": 12, + "selection_rank": "0e666ab1219416c4d077a968c59fc89eb16d12432d4c2c211421e7675a6628e2", + "source_text_sha256": "1c9d5c90e7f754d3ad264e0830f21e50f2f8c094dd52d16a61c9242f31ae7841", + "prompt_token_ids_sha256": { + "s0_eos": "314c3ccd61fcd63856f2e8424d72c418f3cc7d305d44201ade899c194e497993", + "s1_eos": "4a0664887d2809f5ec718fe0c13c23a650cb4e98cbdc923a6b76c4976be2205d", + "s0_period": "2a5c60ec398252f0622263fe7984ecc108ee7568b133a5770e24b080449e7dc2", + "s1_period": "23f7a0583987d45400e7ef911f5bb633db1ad81c07ba7b050ec2b7958f2b8989" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "HumanEval/70", + "domain": "code", + "domain_index": 13, + "selection_rank": "0f95cd54ad35edad763ba9ff249081d4b0c422bc4379ca91ce3a8e608919b54c", + "source_text_sha256": "a668134355e83438ba8e62cd1b442897d98d53cd5190932b73c948d1551d029b", + "prompt_token_ids_sha256": { + "s0_eos": "3b4c780e966fe38eff8fbe400f04e5fedd19bd0a9c8b56386c6df454f8e4e5b0", + "s1_eos": "90e4725519dafd9b93429119a5ae7ea25ede7917c2b5296eca93ab2e6459de68", + "s0_period": "baf86285b628930addd3efb3ce74c06b2cf31589a24834d8582fa31ea09e6190", + "s1_period": "c0b6975794a70c74eb6b93316b20d966cf25bad348eae6f97c8a58d5e4764026" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/89", + "domain": "code", + "domain_index": 14, + "selection_rank": "10bd0378740e88d6557677c66806363a8ab113a310e775ca0820fa7478022d1e", + "source_text_sha256": "1049c3240be776d403e64cd3b73857a6235f079d55024312ceea611853b6ea39", + "prompt_token_ids_sha256": { + "s0_eos": "78a1a74505b355e8ac272d6d0e7393e2f4291195521c24cd12aa84c0d40d3932", + "s1_eos": "800961bc065356f64281dfab75e73eec5eaaf54875339d73405a4fe91c1e2482", + "s0_period": "55c56f5529a94ee972f831a4e1694db7ecb52e7542b5796018f3aa0716197b68", + "s1_period": "b786f5b7457aabfb965b5237c359e5930ae5258a612933df41bef7dd6009c0cc" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/9", + "domain": "code", + "domain_index": 15, + "selection_rank": "11e369155ebde83738eac8a74152f2d999a9d4a51b1813b7c8b790adb89c5131", + "source_text_sha256": "706c5a34dddbfadfb6f6996b8224f7278f9687d562121b60730271d177e03d29", + "prompt_token_ids_sha256": { + "s0_eos": "a35480bd447aef02d0cd01b915cf45d6d888327d5db81996bb0957a22804e1d6", + "s1_eos": "cdd84214cef7754b686416420f6b143cec21c83d238c01102219a4e846bfa6c8", + "s0_period": "901b7e97b26ad8551a40fd27cc2373b5ac60eaa8e1cb198baf5564b389fe1ba6", + "s1_period": "1ed5aa3ba90ee6f3801de4ae7a31721c2c3e7e4ae109a05163f6522638ae8151" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/53", + "domain": "code", + "domain_index": 16, + "selection_rank": "11fb0257fece6b5a2e3ebfad4f913edb55a12e308d703a29ecb51111007252d7", + "source_text_sha256": "609eece5c26c4e3f26df35588daf4a78508df124f64799f17d0553b17bf2be63", + "prompt_token_ids_sha256": { + "s0_eos": "a887f4b746abfed282612212f14dbbc8239dba7b71577ceb4185cd8ef9aa7920", + "s1_eos": "fa7bf486291c89f98d6bc376bda54c7beefa31d3eee26bd0a71d858c71e84d1f", + "s0_period": "69549e7ce68a8c18538b9c4e73722b41a90ce7ce30d4e68cb1d49e915d28c96d", + "s1_period": "dd90eb12c89f0c984c3d038aa866d8e04c879407de6b1e96d3e2f7e386bcf498" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "HumanEval/145", + "domain": "code", + "domain_index": 17, + "selection_rank": "1262afd7962fdb524bae1c88f4ea10ae7820dd4a7a04b7f978acce27f8545dfc", + "source_text_sha256": "8dd398d709ed6d32a38ba15a1a4bb882b08ee0637b397c5b8426e7654825ed06", + "prompt_token_ids_sha256": { + "s0_eos": "6328a259ec67ae02bd8d9b055fe9b587fe0a7930717facc1d600fbf60bdaa0b5", + "s1_eos": "026e05eb4a37af766655aea6bbce07c3f2bc1760c85a6c9b54a93c9bce0b2b92", + "s0_period": "759854e60527a65f28d60edb1efa9835f2c3d25f84a1ca3087f747eb9ed9d2b3", + "s1_period": "19908f2be1ec36aefeedcc31f3b08beaa8f375acd425d3232684b033661add06" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/35", + "domain": "code", + "domain_index": 18, + "selection_rank": "147ade420aa25ee992c940020f0100f2f637b102ce075d6659cac9e99aac5f38", + "source_text_sha256": "f61f2f52f5962ff528b5fda83429b3a9da557e01877b49a0b622c199c0f78b5b", + "prompt_token_ids_sha256": { + "s0_eos": "198a635f3ff0f2dac6febc7b7d5208c84285b39f18b1bd043aa8efb8f1e37250", + "s1_eos": "ace06bd48049a9633d911405047bfae13b3e198da6cd8acd68cef618aeb10a28", + "s0_period": "fe5158583cda206e673d2d0b75614506d4f2a47c60504979619a175968d95a79", + "s1_period": "00fc2a7b46e194df9c2a57a7e6f63874190a7552ae36453d7f21aaa85783cc53" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/103", + "domain": "code", + "domain_index": 19, + "selection_rank": "14cb89bb17e2bbe27f2f4ae91dcb86866ff9ff855c1e5c701292aa6c434a4a51", + "source_text_sha256": "82db1a09d55321f17fc5c9d69ef96b9ae24bdccd6f845d6c91e5969a6b0caf3d", + "prompt_token_ids_sha256": { + "s0_eos": "abb34faf64df8c3560bdcf2f14b44cc6ab6beb9ffd9d00185725b16cdf0faac5", + "s1_eos": "98b973f4e474bd22438d948bce0a087497bf870d1272badd1d5892e26b4d7442", + "s0_period": "c9a00420e743b45663897718741ddc3a74badaaa8e39184fbd2b7ee49a167498", + "s1_period": "2aefe0fc2301c27fb133c9aaf56e3e9238590664b28617821447385e974b619c" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/25", + "domain": "code", + "domain_index": 20, + "selection_rank": "18fe53834c71c01cc9670bef92494b340e9cee5ead1927899fc7ea8a6120b031", + "source_text_sha256": "12e9603b0cdbcbb862e3a790a25dbd1478ed55b331ce84c9bf79865337a5d17f", + "prompt_token_ids_sha256": { + "s0_eos": "008b5386771e660cc5e9627282decda25c90e5a9b75393fbb551eadd081a2000", + "s1_eos": "d659f8c478bc8a6ca5cc4ddd02e3979744a43a58dd1d70d741d9cf66b216edce", + "s0_period": "b59fa6f1b0f5dcb10d292135a80414c958374014497f94c641cf9f1c6c1b65a1", + "s1_period": "23be5ba31d23e77982f9e252634cb614cea4d956ca7d6fe8d6155a9ad8e2925e" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "HumanEval/98", + "domain": "code", + "domain_index": 21, + "selection_rank": "1976fcfcc74e22e8001546540ff7297f08401a27b4868b8748bbcedd3cb60baa", + "source_text_sha256": "11c1bca6656484296abc251387766d4ffd8aa560d1eb56b3949e4d45db10596e", + "prompt_token_ids_sha256": { + "s0_eos": "8150b8ad01422cc6c29862faeb47d565642fdca635db45f5dadc69e95d1a945b", + "s1_eos": "d4a28e727aad7ff1895cc30ec4847d76eebfc5116e7d709f1344de65bb4b77aa", + "s0_period": "6d960563ef9915f42deb3647f839284d232650c00ec5227ce06a1e7092b08cf6", + "s1_period": "b28e5bd00cc4f2460920af0104ec3cb7bd2bba22191957ad421cdee22c559594" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/115", + "domain": "code", + "domain_index": 22, + "selection_rank": "19f11bc8d7cd0f17138960b42a559004aec5c51b92273382be6f4eda53b027a0", + "source_text_sha256": "7fec019bed5056b0a2defe9c3716cd547b81bf4472b02247eee7aed7cef20d82", + "prompt_token_ids_sha256": { + "s0_eos": "6c9f7961965fbb26230a83398ed310ea57fb82187d6b2e30a4d17c35754baee2", + "s1_eos": "bd67206d07f8120b815b449476b17850d7aff6c6f4832f2963d47090a57c2ecb", + "s0_period": "2b8a739358f2f0f729be134ac2aca2db22e60f0ac14d77eb14a7dbf7fdc53011", + "s1_period": "089d593f31fe504210bdf7c02267774f2d1400ee60eb1d0a6f42cfcfe4e4f40c" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/10", + "domain": "code", + "domain_index": 23, + "selection_rank": "1f38e7c9bd528dba6fb5f380a993ced57091a5eee4a68cb07cafe93b5e60af52", + "source_text_sha256": "ba94307a7a0a8d7c490e889c1daf9f07dbc81f3a39d598138c181c212940e49a", + "prompt_token_ids_sha256": { + "s0_eos": "364bf44139c0462ca4bcc49e63ebad500338a9ad08d3e9cee8c02177670fe1ff", + "s1_eos": "fdfa1d03266a6170a3db04066859df942073f5f826ac2537c2802ae9c9eb26f7", + "s0_period": "66d40517ba20b9a028d4cf38773c7b8554c7d0c53e9d9d6c5fbb2eb363cec6ae", + "s1_period": "adb7f6b45b42d9c1e622057ad6838b894f76695d6643779742a03443893c8cef" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/141", + "domain": "code", + "domain_index": 24, + "selection_rank": "2088e9549375dcbefdfc7beb3bac56fbbd0d1d18dc09962421d2e3caa33af6e7", + "source_text_sha256": "83dce9f11beb7fa7a38f51846361581bce1775c55debdc2d8578eee8e7fd2c13", + "prompt_token_ids_sha256": { + "s0_eos": "a028dc5a7a3590901ced6cc20b746ef23da96688cd22cc83896d96c3b95a87da", + "s1_eos": "cce0c490343e8eb7cf309e804bed4e572faf1d88efd37e65747a10e4ae2ca3b7", + "s0_period": "f5b0ae39329c55899a783d0a7bebed8a34fb3eb508b6e09cb82bca0b28e9c7fa", + "s1_period": "0cbf2a9644015e4f6fa0e983ccf9418298167531be661bc0bd6745a51c4223cd" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "HumanEval/91", + "domain": "code", + "domain_index": 25, + "selection_rank": "224e1881a15e5af9898c4481d2d4d42194a182d6b2359d760c5378389d0a0cf6", + "source_text_sha256": "e0ad45711e34cf692a9bb9a0940d1514add9d88e46b98a78a2649ec161f53d65", + "prompt_token_ids_sha256": { + "s0_eos": "8718d95258b5699021eea917f8e988072cd3693ede85617b33641d60fae7100f", + "s1_eos": "c7888672ceb546468f7d3b13248d060ace7eb3f52b0a2795096a83346601d9c1", + "s0_period": "60134f0f46b5ae9f5f7e1e133f78ab7723f65b7395c48c62f258dacdb279e0d2", + "s1_period": "4470c9e0df5af2045c5ca1e375bda3dc6a5624b950baf175d6da85ec6e8cf191" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/122", + "domain": "code", + "domain_index": 26, + "selection_rank": "22bd978cdd54b34eff1d494b071983b628871d89af06188150a80f16c492bcd7", + "source_text_sha256": "d725bd96292ea57f542b6762d55dfacac2ec7d08907941ccca81aaa347ac053e", + "prompt_token_ids_sha256": { + "s0_eos": "3139d57c16d5dec6e7860dbbab99d451f8c53195ad5f54bd55868c997fb2f8d7", + "s1_eos": "50ce61142aa5cab9f7372f78a004c24518b669940d8cc9fc94fdb726eff92dba", + "s0_period": "fba3772d1945be22f53c30a41d32aaef4958f96a555a22991d964d8f66d175fc", + "s1_period": "0d81569eda9f3b8c4e48e24e8b04affb85b349489c1955037f51bf4a16933020" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/75", + "domain": "code", + "domain_index": 27, + "selection_rank": "22cf87a8ef93326fdcc2f72d8ac20f7e24a1f76d8d507b22fa7ce414bbd61077", + "source_text_sha256": "453da210ee2432b4f676136ec260c5d3e4208dc1e7e3d7a5b1d828e626a41eb2", + "prompt_token_ids_sha256": { + "s0_eos": "13284422826c13b4a05bc509ef53724cb0f2dc7d8914c160fa63683cf8307d3e", + "s1_eos": "d274f2b89aabdb9f2285d1e2233ce8d00c69638581eb73b21a88a8ec28b3d349", + "s0_period": "5b73da7a5a302a8566fd704cf0af807bc31602a622f6d6eea086320c92052c64", + "s1_period": "18ae45ad308c0e140eb0df03e0789275dd3ce69a01b134e9fc15f3a34b0882eb" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/134", + "domain": "code", + "domain_index": 28, + "selection_rank": "22eea753219750e26ceda942caf857620485120e116dc434412c40471bcf80d3", + "source_text_sha256": "cd08bd2eab53c655bee6f670158d1f3ccd198e2500bfed1a0fa3b9dfbb28d4f0", + "prompt_token_ids_sha256": { + "s0_eos": "116e6cab53680405e0e482419ca81140023c6313316f8c266251b7f76f9782b4", + "s1_eos": "acdcd2f18ea59c8d961700d2f6e4c877bcfce7a8e896911e3ddc66721f3a3224", + "s0_period": "c21c2757d50db4f53b650b06b6db166236bce49dee2b83cfdc21cad160a40eb5", + "s1_period": "3c12352201902e861c1a026362bbd0b5330a0fff73fabdd65ba4d2292f1a9502" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "HumanEval/142", + "domain": "code", + "domain_index": 29, + "selection_rank": "2558f93c2d7e0b026c8427e12ffbbe04dcb9ba6aa37d3f5baf51f6aa67caede8", + "source_text_sha256": "9a63a21d2c849c05ab21dc8cb7c448a5488a58d26ecd64c3cab438e7d7c759d2", + "prompt_token_ids_sha256": { + "s0_eos": "ee9ddf7e4a819e4799d2c6d28061ccbd377005f8351d38d8181452d74ae96b7e", + "s1_eos": "62d4effe5e30539c035f77b76838a62bb6f3ddc6a4c53de09b0667cba9810408", + "s0_period": "500c8c34bb6fd0b5d77308bf8980581651e00190e4f538f9439be129c167bc65", + "s1_period": "46dda796895638345c03a6878f949933f81cbfc0ad7c361e6b949cdee2ca7fee" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/149", + "domain": "code", + "domain_index": 30, + "selection_rank": "26c65bd4dabdf18fe88f4b40d05b0830d3a46b2b6628cb210f767980c49ba381", + "source_text_sha256": "29bdd295d2312cd6be141f31c1bda08d9165274526c3d0ab5e5ac7b3050ada40", + "prompt_token_ids_sha256": { + "s0_eos": "d08549c7f49b37e5f0405e32152a60ba8cfa0331745b799504e54cbfb04094c6", + "s1_eos": "c5aedd3defba8d749550275c4b1f4815b4329e258f9bd042049c8099e4cd48e1", + "s0_period": "3f2abdf1d60c4502882418535d6dfe59b88627877f6c3f5abece3740bb06b075", + "s1_period": "8f5ebacb99a89be903e6476a2c0e2b45f787632d62d6682829b7617b50752631" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "HumanEval/102", + "domain": "code", + "domain_index": 31, + "selection_rank": "285d2858f6c6758e66af36954d25c5cf8a846e1f7f2cbe1b11c5ecae0af80263", + "source_text_sha256": "15652ca300e14ab83ca395d57a5b770c1d13e4aa2a900b10d6aabc6a77d05223", + "prompt_token_ids_sha256": { + "s0_eos": "10f426b1abd51322bf70819dafc44af8ca8283f33a1ac3cf4cebb1f922917180", + "s1_eos": "c12a2e86abf45c7e729f344336629cfc4bdee4655f99b74a997a59f76260fabb", + "s0_period": "f151d974953a5013bb557151a46d8758df5ff22051a046abc3e8e3692967fcf5", + "s1_period": "3614024f4cb3124704d6d7263a29f71fba3e534b9477f53cd77881eaa3f014c4" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1069", + "domain": "math", + "domain_index": 0, + "selection_rank": "000946b459ce33a645f403b9d781a7aefd81f5568412149f4588381fed55b1d7", + "source_text_sha256": "0cc2ca2bdd6ee8d664eb1197e8020bac71c25ba0ce52d813fc4adbf08df22378", + "prompt_token_ids_sha256": { + "s0_eos": "f18b1ab81945738da2e32e8b125a022710ce45c6143b45d8ca13df6469d34ea1", + "s1_eos": "09336f425a886a0fbe460639523aac22cfaa1cde4e4bb090246da3a338cf6b42", + "s0_period": "66afcd11149d1b34a62305b2278b09dba96ce7f47c22b0bbdc7f7b17fd3f58ba", + "s1_period": "003ccd9768634d4ca3dc8ac537a0073882b7e5f6aea85cd33b53826700177a2b" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "gsm8k/test/1228", + "domain": "math", + "domain_index": 1, + "selection_rank": "002201d4cbc0b85d338597dcb68c3ffd4d00ef12b7b44e352c5bc1911d6fa984", + "source_text_sha256": "3931b6363f3a060ca7cb0774662127e64007e9f34ce209edddca77425e20a479", + "prompt_token_ids_sha256": { + "s0_eos": "83658d07cb0988b7cbd74a81e597117cf0e09235294b78873b7cdc8b7da6805b", + "s1_eos": "73497c3325f69f2666625cce2bbc77f5f0197e3287e256bb44aeb6f6cff83559", + "s0_period": "cf5d4d0156a790e702203557d710831ac6fe7f2e709556f3e54c24882c616b16", + "s1_period": "15fee51aa30887005f602a8a7aa9dc86e4470e8f2a4ff81ae49062fef9ffb872" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0144", + "domain": "math", + "domain_index": 2, + "selection_rank": "004a245538bdc42cc175a178c73dfaaaebc9713e3215ffcdfb9400fb63b3a941", + "source_text_sha256": "8f2ecd574d8c8738ec87b6ba3a3af24f9cbf35d832733c98e65546f7ee59725d", + "prompt_token_ids_sha256": { + "s0_eos": "6024c63249b3dc8eb52d0d90542991b0c4ad74a229d7911c1c177d738fb7ad51", + "s1_eos": "87db024f771ce0454066cff4046274c265284f2d4d804dd0928c9e0234002e0c", + "s0_period": "be880ce76c36b376deaeff9209ad0689563ad54f9e9a072d93199d1482d24d5f", + "s1_period": "37f350b9cbc99cc7ce9e324bc5c85d3dab67a02a36c392b9fd02fe79405fe05f" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1251", + "domain": "math", + "domain_index": 3, + "selection_rank": "0056e66a635febc5b2ace25221ed8fe25366682898fc2d4d0132b30812f59e30", + "source_text_sha256": "a45936495f064c16cf6621fe21bf0d84f54f5850bf6bfe19ced45b30c591aa36", + "prompt_token_ids_sha256": { + "s0_eos": "67a88758cbdf61af3b2742efb3bfc7f22ab6fb0851e8ee801cadb0aefba3a504", + "s1_eos": "2fa687b5c4a1176cad1b14b25ea7619bc3aa21983182fb774527fe69d3489609", + "s0_period": "9f20020cdaf7244aeff4b240e15cf332e9a4cdced0dc35cff49a22e927b61b11", + "s1_period": "06728f575eba83f69c132645e4bad9ecb86435e45e254327d69aa08a0b8566a4" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0676", + "domain": "math", + "domain_index": 4, + "selection_rank": "007dc3953514b93637a47cf96da05c0eb643a6d67337f9513efc328c3a50de12", + "source_text_sha256": "ce589ecfd3ab0f3d1d03835a292fb5029f9d91cf434342da860f0fd0502e8323", + "prompt_token_ids_sha256": { + "s0_eos": "c890b320355dc2d0a1a578821e4ae36a57cb2653d4cfc03c6d1d20bbe9a0f391", + "s1_eos": "f8cfd874be86837cb6f805a22e5d04ecfd596167065b0ad4890e9513b3c4e422", + "s0_period": "a30d5afc1522ffa514ddf5b5bbd714da69889724aa36f18b0129229a036db715", + "s1_period": "1dfa070f5682e190d77a0998e293dd4ce3576ffc51a865e63929d65970d03a20" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "gsm8k/test/0126", + "domain": "math", + "domain_index": 5, + "selection_rank": "00b20ae940ac731b073dbf7adae91f69acdb81ecf4d92a0727f9edbdbe14d634", + "source_text_sha256": "d0798145442485f3c0cfcdfd90a7ce1c5d2eb7496d997bc8c3423c1b8e99e49e", + "prompt_token_ids_sha256": { + "s0_eos": "42c33b13cc1155eda2bcf4eac8e42640a59b1270ec43212ad0fd878357c0b8de", + "s1_eos": "6c9296e0ce46e6645bfb47f42ba41e054aed769507435a4fc868de56efbb40ea", + "s0_period": "561f1d96511bc71e1250dc3478f6dd420fbeb6a0eeb4722ff903a438086110f3", + "s1_period": "6bcfc9f0ebea0273a2a41b2f480ac8ff688aed41f87d94cdc4394a8afe5ce89f" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0438", + "domain": "math", + "domain_index": 6, + "selection_rank": "017a97d03b63645963eea1c1f6a69b071b41c2be4d43e3fffd4f1dffa6986cdc", + "source_text_sha256": "0246c431c39aa3351713edd9f897904b632abc5414727b7e0bb7ffb51e0804a1", + "prompt_token_ids_sha256": { + "s0_eos": "3736af9c3456583b2249d50aa45b27bedc2a90f143e77201d73682f8108aec73", + "s1_eos": "ed42a4cd317988656e1ed154fda077a6cdbb369c96cf8aae4ba37b49144d876d", + "s0_period": "e882c307ae0e4221c05b08099d3e0fb2943fc6416b2ae513682bca73d8795987", + "s1_period": "5a99e7d5af946a72524c9df0a416cfb2e72e00c706333470840798191a829fed" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1299", + "domain": "math", + "domain_index": 7, + "selection_rank": "01905ee47b39eb58804378d2e1e469bfb676f2f1524864076667215f0f12815e", + "source_text_sha256": "d4350e7f863b05ee6fc0dc0e92fe04815e183172553fc1674fbef33febb33971", + "prompt_token_ids_sha256": { + "s0_eos": "7d7082e770e9856f30414c0c855592e7fcdfa6d2616e3f1e78c013b9edc98d5a", + "s1_eos": "0907bc1bc4fb9702bfafae016dcc58e441b02cc0b0861991b37fc4820b085b3f", + "s0_period": "2a2e1d480678611c601a5c98bf916de7d95027baa63371a4a2c12f563887a3f6", + "s1_period": "de4fb3a47ca35a068920a059973a4fd2b74575f8c9d5684a260b2b379e71e14a" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0546", + "domain": "math", + "domain_index": 8, + "selection_rank": "01ebd6fb979d45e8755a234bffd810bc5ec36423c0fbd8bf344a799e5e0993a6", + "source_text_sha256": "8619c48853d9610e152a7525a70d157c33e2f3ec1f9bcc8b41f927324527d5f2", + "prompt_token_ids_sha256": { + "s0_eos": "f541a80cc280c6895b3b4b2f8ec5e58f75cbaebc4af63ec907461d352a5cc022", + "s1_eos": "84ddace95218cd7fc6cea61ca3d9ca463bb779212ae5668cdd40fb503dc9dd8c", + "s0_period": "82b528af177d489acf469e86715f99f7c9891b15ccea8ff4ed0b360a4365ce47", + "s1_period": "069f2515bc9dad73ea6a5da23b53f7d25906169e665ceb5e42c5d5fcac84b4f7" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "gsm8k/test/0537", + "domain": "math", + "domain_index": 9, + "selection_rank": "01ed2a6eaa7e8c6439db165853d4bcf48477213319beeb260fa08b7f3d7df971", + "source_text_sha256": "a9af7c44056527232865dbdf9952a307e8c3f2db4685e5abe30d0281646780cd", + "prompt_token_ids_sha256": { + "s0_eos": "71a94bf2c87b0cd8f9a7591940a09f6625c2ff5495f6489361be29e26a9f56d4", + "s1_eos": "ef240167f5b53338e4abd5080e4f0643cc05db10bbfb6d0efea0612b93e48e2c", + "s0_period": "582ce43df4e935e0f6144145969aa9ad1c86d638f2c12ffa74f9d3e488fe17a4", + "s1_period": "515e612bf2c8836ed6cac0fddf6218c58908e436c631987e680436b0566e24bd" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0226", + "domain": "math", + "domain_index": 10, + "selection_rank": "01ef79122976f164519533009f7bbb5f72fd91d9c90f6c5a4df80ed3091d303a", + "source_text_sha256": "d58e5838fc7750bc87cf98d3888c5cb7bac568a0c7969d1466ce5fd6acb956fc", + "prompt_token_ids_sha256": { + "s0_eos": "d5867f77ff3258595f13a65b243359568e11b130297912ff597d2ea6d5468144", + "s1_eos": "3c3083464ffcf2e6c3c8855a3f50db7c8cee4db7605c5948b52794fdce603583", + "s0_period": "de593445d8d3ad3162890b148a5f2479c034d8a276b2fd8e2f62c1ec6e010ccc", + "s1_period": "9301e6039d47d3aedd7d42de0f5ef80e97b0277a44f48ae593e74f0e0ce14b5f" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1128", + "domain": "math", + "domain_index": 11, + "selection_rank": "020803c399cd482ba6ad3ca246e7c695d62e95673fd612481a8f9ac21f646e54", + "source_text_sha256": "9105ce5963e380fed4744960fecf97a99f44de151f6b4d5fc2c48e8817f3225d", + "prompt_token_ids_sha256": { + "s0_eos": "52727d484c42c7e94ebd9af0a93a6e8c915fd12397ce65dec84ceb9162dc9a90", + "s1_eos": "264621d37e45d1fd34a1626589aba9c43ccafb4706b8ff74a80e33f24587c94d", + "s0_period": "e69f030f2d8503473730ba3a9f40bd7666b1a577f5945854c5b3df5f2d6e2ae1", + "s1_period": "ebdb80407ab130cd3488cc0f3f0b28cd87420744e69fdcfb92d3ba34d1147e1f" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0760", + "domain": "math", + "domain_index": 12, + "selection_rank": "0208e9afa6cd76d86a6e08fecb0736003ef967fa0a434352c3c4053fdd11b5a2", + "source_text_sha256": "65a55bc0c16200440b69c4cef6334f0a33798b1fd5a1c079300911ebccb75a06", + "prompt_token_ids_sha256": { + "s0_eos": "8ce20b40bdb99e4a074e2aa49d6387fed6a87cbd3a08f9b9358cba3d5309a6d4", + "s1_eos": "ae3851beb4032e4f9976924634edd56ded0ff6e8410da3bf6b3675316556d7a2", + "s0_period": "4495e2738d8ec84efe9cd08ad4d7cc7c74ab5ce3ce1a4a48e926d24a270b76f4", + "s1_period": "3e4530b8653c18bd67060665de5b37a92640ebef9f9e958c92f112344c525053" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "gsm8k/test/1173", + "domain": "math", + "domain_index": 13, + "selection_rank": "02a7c343f3c4a74787465d7e08787db473bdd1ee6c296341979585eb5da0d3d8", + "source_text_sha256": "6206bc51a78315ffc176309157e0bdb07d5eb50510d21e8555c6370c7fce9df9", + "prompt_token_ids_sha256": { + "s0_eos": "7a07020f6f73b0aab44a10ae7a77d4687445fd8e79aa4c8544633122ceed1bd4", + "s1_eos": "bf48634fda477a400a0fbdb7fe3737b17a95dbb86290292ea7b6a910cde07cc9", + "s0_period": "4b6f1cc014d4418ea28415d5348e53b3abe6a30cf6560ee31cacba2a1178ae5c", + "s1_period": "5cbbe5312560d7e281efc607ad55f10d4d42518465428ba955d3112eda1f4836" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0452", + "domain": "math", + "domain_index": 14, + "selection_rank": "02d8131120fedb69911cd70aa26c56591fc5bcdba6fd3157fd571c410d039988", + "source_text_sha256": "5102790b75edc5b1a8d08709e3784fc762dabba766ae03b14828bd18bc7cdbf2", + "prompt_token_ids_sha256": { + "s0_eos": "5840d636471247bae3184bd15b5bc89a0994f07009438bbe41a6051de077b1fe", + "s1_eos": "83d9af33a6fc56b45ad62054d36189919424c11de77c771b50d96014b2eac256", + "s0_period": "e14cdacb293045d67d4ae47e9a16ac69094e6fe77bb8cc045d2a5297059b0fe6", + "s1_period": "67bd743ce9e7009573988495878fd6885cd2c116e57b5e9fd269a4af17ad8a18" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0357", + "domain": "math", + "domain_index": 15, + "selection_rank": "02f701d64bfd4ed535275bd8010260c8884dd2e0ed21a5f102084e82c286f84f", + "source_text_sha256": "dc1b264fc4aefafbed621c9e844315a6e512d5d91ccbdc591d21512a2ab9e855", + "prompt_token_ids_sha256": { + "s0_eos": "304784cf8f1dbab4fac0f830bc409aee7b73545a306949244474cebbacf45f02", + "s1_eos": "15b1516b88dff7216e2ddc793dfa931c4d97b3ac97f6737c1f7f870050107e35", + "s0_period": "4f770a111c25c8cd2355a7546e19413e9e82ea62fa00c66b9ca5a50001b54033", + "s1_period": "b71eac63a204769d883c3a7211a0c2e9f9e2f154492e062e8e2da005e4ac9150" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1050", + "domain": "math", + "domain_index": 16, + "selection_rank": "032013ab41bff5a0bd0eff093a92ef002a0dec6e348ebb5d723e58176574151a", + "source_text_sha256": "8af254886db4666b82ef6475cef3644fe4ed391d4eae588c13b19173d19f75ad", + "prompt_token_ids_sha256": { + "s0_eos": "8ab8b472fc249ccaeeac8319981b0b0d3b80f0ec92f530f3730a3f1287b791ed", + "s1_eos": "8db9a2e97eed8c7b85434cd86d557e94133a293a7e06a189095c8e2c79bb78ce", + "s0_period": "1a8b7a31afee52ca22a1cb1d9d0dcbfd47feb024f158a300072c17d13f11e51a", + "s1_period": "b4569da5b0dedb6ec0086debeaa21484972ddd1c7dee632b12251396a92e279e" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "gsm8k/test/0951", + "domain": "math", + "domain_index": 17, + "selection_rank": "032d853068a127a2d89b7aea49ace29b4c161f55c9dc4beec570b38e2f173406", + "source_text_sha256": "8fd6525f5b25a51226ed5c53e8d6fa871061e7d766d51d66cddc37eade3888ef", + "prompt_token_ids_sha256": { + "s0_eos": "e6c89bf1e8fd4d01d1d2ca7fda4018121c8ee15955fa233bc19cb230a9a96ddc", + "s1_eos": "8962f074c0900700d95b84de226a16c54fca06fdcd0db2682cdd4bf46657e8a3", + "s0_period": "19581e485e193a2b090cf037781ea34221a42981b0bf113b8f2517bd51aa2a1a", + "s1_period": "8b1f492f8de1c4456fb4d58d609949f503b0f1c5936dad43e360455247233237" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0906", + "domain": "math", + "domain_index": 18, + "selection_rank": "035256813697b847333e8d1e657ea0a5a2793a5f4fd79ec170bad08fc63feba2", + "source_text_sha256": "da2760f4f2f38c084029728114c2017b9ae49d0bf21f8846968cd705038e9cd5", + "prompt_token_ids_sha256": { + "s0_eos": "1027b164359b9f3f47bfab7a50fb8569a683fc12744ff9855d50080f93f60e9d", + "s1_eos": "ff186a4390d6d398ae9143f515bf601aa9618c5da58564dc7da822411c1c9983", + "s0_period": "92df0af9657b53897995f5990471c294b895f1bc81ea145185dcda1a09e2b95e", + "s1_period": "cdb3e5057f568334b37ba88e4bc9c2ea8aef2d20a72e915098bf9679442a817e" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0092", + "domain": "math", + "domain_index": 19, + "selection_rank": "036fc57b14ee989e00cea584e28add4f5123d84b4e632c7b777aa2b410ac2c4e", + "source_text_sha256": "30e10116f95bb87840ace44d50ad933c09d0937a25baa8d8f80c76bd7535350a", + "prompt_token_ids_sha256": { + "s0_eos": "a37064b7eda6a5b5d6ef875fa6f1edd37376ad88469ba4f4f667a8076c22f775", + "s1_eos": "76196d53d00bc859ac346e9f7ea59efe96685a34408ef2f7b703d9b2ec7e10e5", + "s0_period": "df8f8eebd79aa074157eb109425ca8b1d055dcd8ae92b75ca6a592c7dd984c21", + "s1_period": "10e0782781a30dc9f59f982752337b6fbeed04f11d4faf03e598e835ff021b38" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0372", + "domain": "math", + "domain_index": 20, + "selection_rank": "038cd03fb9104a796adae4930a3b7eae24ea07906d0789b3ceecda2a338b3714", + "source_text_sha256": "7810cb6f7d1e2ec732197a85f05988002b179c41adfcc03895d287e4568c53b5", + "prompt_token_ids_sha256": { + "s0_eos": "12dde598aa845bf02694415058832208c0ad5f75147a84855daff1885325146a", + "s1_eos": "fd202f3e2769ab73d59ce5e9a5cb80f5a239cda4a3f1106eb6a2cfa208086ba9", + "s0_period": "ae25372851d8574023b245924596f8e0b58e52822469f0c6bbd51c44cde10619", + "s1_period": "cf536497903650ba9242266841c07a57c24b4f0d0677152167444553db1dfe2b" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "gsm8k/test/0361", + "domain": "math", + "domain_index": 21, + "selection_rank": "03b08b85b81888c5fc1033dfd9c132aa147d5bb85d486cf32917c01a8ae5f54a", + "source_text_sha256": "de5d935c56ce69df276b4d84b4af6a6aeeea6521cbae8a75e66cc2b106be8550", + "prompt_token_ids_sha256": { + "s0_eos": "9d39ec1e0da4c43efed250caf61b4f258dd438a60553cfec7b01d304d31adafb", + "s1_eos": "999cd4ffd587795760546118a06c3281736dc8f682e9aeda8dc6b8e559c01161", + "s0_period": "47cbba12fe460791b8fa583ca1b201763473dd775f0fd7a2b6ea5174ba77f5db", + "s1_period": "fa7f85cd8342417cf2eb0a44b5df4da1d079c962b0edb58800e6b8051600dd8a" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0733", + "domain": "math", + "domain_index": 22, + "selection_rank": "04307c4d732dcd9b5fe573fa99772baab669acca379a641ed9df72ef2ed18dc2", + "source_text_sha256": "26e868b2ce725fc01164e8efcad60d8de755341dde1b5bd001e7fc9815f43197", + "prompt_token_ids_sha256": { + "s0_eos": "978117c99ac8e08204be65dd4a6b5646f008f5c282fccaea5678fa4c3a7c52af", + "s1_eos": "9b25e7e2923fa80396876c50640cbd85d4fd016b1dc9299abbc2daf4500dd5c1", + "s0_period": "4ee821d82e8d87c60641e61e3fe051a53f7bb8da4297b9423412df58fa6ef549", + "s1_period": "a15e5895e75c6b1591b640c9a927991fa30ea5f93e0c1413b83014af9d4befc6" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0687", + "domain": "math", + "domain_index": 23, + "selection_rank": "049c5780de56bc8eef319b50fcb3db8b66919b5af48f1c335b4ef15bfc0dbe13", + "source_text_sha256": "8442b01bf873cb623d48ce23f5616a670fbca07b2898cd258d6fe9de0ea1890c", + "prompt_token_ids_sha256": { + "s0_eos": "9c13a3554c65bbdc9fa30d3cabad7eb7b7b26c1a47f205b879f96e54533c08f6", + "s1_eos": "73902fd1a27a1d5e582e7f6f8875515090b4e2176ac622c70e59eada9545af94", + "s0_period": "5e304cd15cf44d3a5338620923e21f78abd6c9564987475f442010eef108703a", + "s1_period": "0cbe33a6e005b8c3cc7443dd57b76400947d002902f9b6e1b99d992b64ee4281" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0584", + "domain": "math", + "domain_index": 24, + "selection_rank": "04a3de37eb106ccbfa9f152f977fccdaf9d81c408628f36b02af7eac2a762488", + "source_text_sha256": "9f45de87481deed78ce0cc5d36b78a3bcf6da872c193524d1ec968613145e29d", + "prompt_token_ids_sha256": { + "s0_eos": "b57e1a8f185bbee5b8c67782aad0df686380ea73509d18a987510035de2e916f", + "s1_eos": "95baf28aa3e9b6bde673683fbd39990c9bc68cd9e81b276f7a6969874449469d", + "s0_period": "752382469a45e49b721b5c52a9d7a281fc795fe0a98351eb102787e73f154fec", + "s1_period": "17767d8339f95439a656b66428b3cab9673ba9bcca2d15c5c2bb52a141400164" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [ + "T1", + "T2", + "T3" + ], + "independent_replay": true + }, + { + "id": "gsm8k/test/1152", + "domain": "math", + "domain_index": 25, + "selection_rank": "04d2980a7e76a708be593d0b47ddfcc9926307894f4c20532fa44f1bdf4a48de", + "source_text_sha256": "fedfc30ae73a9f3a9b142acac32bd214a14407c73079c107076740f8db8fb09d", + "prompt_token_ids_sha256": { + "s0_eos": "fb8109d835341a038d61c4bb4c864b1768b2b65b78e8bec0cb5cb788ee7497c5", + "s1_eos": "fb8562da582f8ff10b0b66588b29f99797d1fab401b024fe26745dd63347d3c4", + "s0_period": "5a561a45f504b224be21fbec6e15312b3c5fa27980d364ab6ff37c524640a22b", + "s1_period": "3f1aa94d697a0e7b09ff9ca646b7531524800822f4135ab2acd284bc85e35027" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1021", + "domain": "math", + "domain_index": 26, + "selection_rank": "04fd8b0beeb0af5582a86211ab7ac01a357f50edb59c6e0a6cabfd64b17639e0", + "source_text_sha256": "d30137acb2e87d30e7d3bebd3a78fe421fd33ed3dad252262c0805ed83b73cd0", + "prompt_token_ids_sha256": { + "s0_eos": "776c7c476253fb5467b2f5f4f2848b110aa81c5d0a8a734ac942735518454a30", + "s1_eos": "c4f23828df7e618026256de39e847ae60fdc7f586e1dca024d06310ef9058a08", + "s0_period": "bacc5126dc02832423e31e973369a21e9da891aa8eb670c271107064c9be7f5a", + "s1_period": "4bf2f6b730620c9141a0668d023aa32b142f172b950eae3ccc61259c07f4b022" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1194", + "domain": "math", + "domain_index": 27, + "selection_rank": "0514f7f9fc1da8490ae0b11068cd65dd3c1ab52e51471ac3fb024b9d56631806", + "source_text_sha256": "403c14d7e3268a34e7db02d081f0111ca60985533e231e07ab8787d3b58d03c3", + "prompt_token_ids_sha256": { + "s0_eos": "fbd53689e2013696720f5c5e00238b3910a483885c759d6b00936864f467e434", + "s1_eos": "c9b6f3a0501f5993263169d3a8335ae791829b8b5f93a270721f8af1383f6df9", + "s0_period": "5d7798232a7c354df094c71470d7e78183f931a104be9c1bf91d34f603d486ee", + "s1_period": "0c7c22898f64ab644d6607258a58d173b4575e362a6ce73f451c90651081a68f" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0399", + "domain": "math", + "domain_index": 28, + "selection_rank": "057d0b8ed707d05f39d4fa70b905e8a3a60efc7f6c35cc3ffd4fdacd16bde7eb", + "source_text_sha256": "bfa0079a8897430238b73ceca009014d55719a9fa8e21000051ab05a8f3eee88", + "prompt_token_ids_sha256": { + "s0_eos": "9be80fec516ddc66691d40176639c448785feb8c1bb69d521da1dfbfd35b3e83", + "s1_eos": "cd28894f20e682b2714baa4a4dec7f7e1f04b29fba9bf30a56ec289759d09a67", + "s0_period": "06199208f8f44a0b9070260b68e10c3863418bc5731eb24fd8077ca1781c439b", + "s1_period": "3ae2b0d27dcb98328b7c3628e872862934134b2e56cec4ada72e7284b308b6bb" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": true + }, + { + "id": "gsm8k/test/0214", + "domain": "math", + "domain_index": 29, + "selection_rank": "069590875c372687074281a45b22de709b7628b30d4cc5f7a53ce2f331c529f3", + "source_text_sha256": "13ddeb54e970018f1030ba0963d8195825f540a6f0eef5e66367a8d4d435ea29", + "prompt_token_ids_sha256": { + "s0_eos": "8c5060bdc697da2bec8c253ef438d5a12d26cf421131e2cb8846bd8e982054d5", + "s1_eos": "cbc818448c42da48997bad65889294ddec0c481d7695045c7fdb82345309b4f8", + "s0_period": "72a8c2af8c0e2af37bbfb91ed2bd25d0b5cc8948c47350fcfc241816099b950e", + "s1_period": "a87ce95bf985d3a12dd288d369f88b1912ac5131649ae55c6fc9529f71b4e10a" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/1084", + "domain": "math", + "domain_index": 30, + "selection_rank": "06c58df5fb898da5f9546423dcfc2087a617b6a06a6c616972f8552e9f7aa489", + "source_text_sha256": "3c7ee3f0b13b459809461336522fcde2dc36c4d5471d24717a62f47e4e313255", + "prompt_token_ids_sha256": { + "s0_eos": "8c823af214d1fb1364776a261f69e41a2fdba56e7727d6df6ce73be8d5df44c3", + "s1_eos": "21ec3767df8846cc1974cd8cc147fd6b1405e7fb9a91126cd0ef9576ba16df1e", + "s0_period": "8dbd28fce4dc52cd3d61fb267cead976b2a701dfadc08484d6368f13a93875d7", + "s1_period": "a91d4d7210a95fa93e8cff0fc884e2cdeb3132d9d679057154cf211294fa3d17" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + }, + { + "id": "gsm8k/test/0964", + "domain": "math", + "domain_index": 31, + "selection_rank": "06d23587754d564944074e12dcaccf0ab91e70a3eafddf832d30f2553730d38a", + "source_text_sha256": "31c70fdf274828397d34824e8342ec13a0e3c45714f337ae0ae0d8aed2272e0f", + "prompt_token_ids_sha256": { + "s0_eos": "04fb0dd44388cfc0cf0de77e5d4c38070914ba2d309b5170b210f4600d8999cd", + "s1_eos": "89925b195626bfe80411cf8665d38f29a715ba391c07a04f5f3548137735b091", + "s0_period": "26c81bf993324fdedbbcd85e3bc3cf85fb468d79f0b2e635d575b92eeeec9724", + "s1_period": "93cbf720500d01268ecfdfd4ae933cac1223d4eb86ebbda4266c1b60c120246f" + }, + "main_tapes": [ + "T0" + ], + "diagnostic_tapes": [], + "independent_replay": false + } + ], + "source_counts": { + "code": 32, + "math": 32 + }, + "tape_contract": { + "tapes": [ + { + "label": "T0", + "index": 0, + "display_seed": 2572353518, + "derivation": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1/tape/0", + "derivation_sha256": "9952ffeec840649d2654d35b9a86c97db3737b70826c366e3f5a7633ec26d0c4" + }, + { + "label": "T1", + "index": 1, + "display_seed": 399507326, + "derivation": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1/tape/1", + "derivation_sha256": "17cfff7e5b3b6d011f4d1b8076637d4bdfb0ebd8a6021beb2a0011a8f6d520c6" + }, + { + "label": "T2", + "index": 2, + "display_seed": 405284229, + "derivation": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1/tape/2", + "derivation_sha256": "1828258525ea1b450d6de0ba0d9ff12f737b048b1cef554d7dd2ba90d25e5f4d" + }, + { + "label": "T3", + "index": 3, + "display_seed": 568701915, + "derivation": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1/tape/3", + "derivation_sha256": "21e5b3dbe506f1b7e9d483318195fc5cb4990c518850a52d40b5eff225491bd2" + } + ], + "main_tape": "T0", + "diagnostic_source_indices_per_domain": [ + 0, + 8, + 16, + 24 + ], + "diagnostic_tapes": [ + "T0", + "T1", + "T2", + "T3" + ], + "uniform_uint64": "first 8 SHA-256 bytes, big-endian, of protocol_id + '\\0uniform\\0' + tape_label + '\\0' + source_id + '\\0' + decimal(step)", + "uniform_open_interval": "u_t = (uniform_uint64 + 0.5) / 2^64; cast to torch.float32 before searchsorted", + "common_random_number_scope": "For the same source and tape, all four conditions consume the identical u_t at generation step t while each trajectory remains active." + }, + "execution_grid": { + "main": { + "sources": 64, + "tapes_per_source": 1, + "conditions": 4, + "outputs": 256 + }, + "additional_tape_diagnostic": { + "sources": 8, + "additional_tapes_per_source": 3, + "conditions": 4, + "outputs": 96 + }, + "formal_outputs": 352, + "independent_replay": { + "source_indices_per_domain": [ + 0, + 4, + 8, + 12, + 16, + 20, + 24, + 28 + ], + "tapes": [ + "T0" + ], + "conditions": 4, + "outputs": 64 + } + }, + "bootstrap_contract": { + "resamples": 10000, + "domain_separated": true, + "main_task_bootstrap_seed": 1364512825, + "diagnostic_task_bootstrap_seed": 963840514, + "tape_bootstrap_seed": 980269685, + "estimand_boundary": "Selected-task resampling bands for this fixed 32-task frame and one frozen tape; not benchmark-population or generation-seed confidence intervals." + }, + "canonical_content_sha256": "42ec988585d34ce9c2a193dd59a30a7f549d8a67b768c17098aaf3f30ca73804" +} diff --git a/research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_PROTOCOL.md b/research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_PROTOCOL.md new file mode 100644 index 0000000..a4cf90d --- /dev/null +++ b/research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_PROTOCOL.md @@ -0,0 +1,381 @@ +# DeepSeek-V2-Lite-Chat 任务 bootstrap 与显式共同随机数协议 + +> 状态：**已预注册，尚未运行本协议的任何生成输出** +> +> 注册日期：2026-07-30 +> +> 协议 ID：`llm-atlas-deepseek-chat-task-bootstrap-crn-v1` +> +> 冻结清单： +> `research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json` +> +> 清单文件 SHA-256： +> `7f5ef75f8e4bade35875e9132ea16f25f7fdb54450490e07cf1f6b2f701fceac` +> +> 清单规范内容 SHA-256： +> `42ec988585d34ce9c2a193dd59a30a7f549d8a67b768c17098aaf3f30ca73804` + +## 0. 一句话说明这一轮 + +Round 07 用 4 道 HumanEval、4 道 GSM8K 和每题 4 个 seed 回答了“换几道题后，现象还在 +不在”。但 4 道题仍太少，而且把同题多个随机输出当成多道题会制造伪样本量。 + +Round 08 改为： + +```text +主分析： + 32 HumanEval × 1 冻结随机带 × 4 条件 = 128 outputs + 32 GSM8K × 1 冻结随机带 × 4 条件 = 128 outputs + +随机带敏感性诊断： + 每域 4 道预选题 × 额外 3 条随机带 × 4 条件 = 48 outputs / domain + +正式总计： + 256 + 96 = 352 outputs +``` + +它第一次允许在当前预选的 32 道题内做**按题重采样**，并且让同题四个条件在第 `t` 个 +生成步真正读取同一个均匀随机数。它仍然不是完整 HumanEval / GSM8K benchmark，不估计 +模型总体能力，也不把 counterfactual `period` 序列称为官方聊天格式。 + +--- + +## 1. 研究问题 + +### 1.1 主要问题 + +HumanEval 与 GSM8K 分开回答： + +1. 在固定 T0 随机带下，`period - EOS` 是否改变选定任务上的通过率、自然 EOS 率和输出 + 长度？ +2. `system on - off` 是否在 EOS 与 period 边界下方向一致？ +3. 任务级差异是否足以让 Round 07 的 4 题方向翻转？ +4. 同一均匀随机带下，四个条件从哪一步开始分叉；分叉前是否逐步读取完全相同的 + `u_t`？ + +### 1.2 诊断问题 + +每域固定 index `0, 8, 16, 24` 四题，额外运行 T1、T2、T3： + +1. T0 上的任务级差异相对随机带差异有多大？ +2. 主要 contrast 的方向是否依赖单条随机带？ +3. 对同一道题，四条随机带带来的成功/失败与长度变化有多大？ + +这些是敏感性诊断，不与 32 题 T0 主分析合并成一个更大的“样本”。 + +--- + +## 2. 模型、软件与提示合同 + +冻结： + +- checkpoint：`deepseek-ai/DeepSeek-V2-Lite-Chat`； +- revision：`85864749cd611b4353ce1decdb286193298f64c7`； +- checkpoint 身份：SFT Chat； +- Transformers `4.41.2`； +- PyTorch `2.11.0+cu128`； +- BF16、官方 remote modeling code、eager attention； +- embedding + layers 0–23 放 CUDA； +- layers 24–26、final norm 与 `lm_head` 放 CPU； +- `device_map=auto`，GPU budget `28GiB`，CPU budget `80GiB`； +- `use_cache=true`，每条 trajectory 的 `max_new_tokens=512`； +- BOS `100000`，EOS `100001`，PAD alias EOS。 + +固定四条件与行序： + +```text +s0_eos, s1_eos, s0_period, s1_period +``` + +- `s0/s1`：system off/on； +- `eos`：官方历史 assistant 边界； +- `period`：把该位置的单个 EOS ID 换成 ordinary period ID `13`； +- 其余 prompt token 与前序冻结模板相同； +- 每条 prompt token hash 必须与冻结清单逐格 exact。 + +正式生成不是四行 batch。四条 trajectory 逐条执行，以避免一个条件先 EOS 后 batch +padding / RNG 调用对其他条件产生隐式影响。 + +--- + +## 3. 任务如何在结果前冻结 + +输入来源： + +```text +src/data/deepseek-v2-lite-routing-special-token-family-control.json +SHA-256 c372c1b03a8b15f615b54ded5d9257a8fc2cdb7728001735d3d4c1d8534af5bf +sample salt llm-atlas-deepseek-routing-template-control-v1 +``` + +每个领域使用其中按 `within_domain_index=0..31` 已冻结的全部 32 条，不根据 Round 07 +正确率、输出长度、EOS 或文本质量二次筛选。完整 source ID、selection rank、源文本 hash、 +四格 prompt hash、随机带分配和重放分配均在冻结清单内。 + +### 3.1 HumanEval + +```text +00 HumanEval/31 01 HumanEval/44 02 HumanEval/133 03 HumanEval/23 +04 HumanEval/147 05 HumanEval/34 06 HumanEval/123 07 HumanEval/77 +08 HumanEval/22 09 HumanEval/127 10 HumanEval/37 11 HumanEval/48 +12 HumanEval/158 13 HumanEval/70 14 HumanEval/89 15 HumanEval/9 +16 HumanEval/53 17 HumanEval/145 18 HumanEval/35 19 HumanEval/103 +20 HumanEval/25 21 HumanEval/98 22 HumanEval/115 23 HumanEval/10 +24 HumanEval/141 25 HumanEval/91 26 HumanEval/122 27 HumanEval/75 +28 HumanEval/134 29 HumanEval/142 30 HumanEval/149 31 HumanEval/102 +``` + +### 3.2 GSM8K + +```text +00 1069 01 1228 02 0144 03 1251 04 0676 05 0126 06 0438 07 1299 +08 0546 09 0537 10 0226 11 1128 12 0760 13 1173 14 0452 15 0357 +16 1050 17 0951 18 0906 19 0092 20 0372 21 0361 22 0733 23 0687 +24 0584 25 1152 26 1021 27 1194 28 0399 29 0214 30 1084 31 0964 +``` + +GSM8K 行均指 `gsm8k/test/NNNN`。 + +gold 只进入生成后的 evaluator： + +| 输入 | SHA-256 | +|---|---| +| HumanEval JSONL.gz | `b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef` | +| GSM8K test JSONL | `3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14` | + +--- + +## 4. 真正的共同随机数如何构造 + +### 4.1 为什么不再只重置 seed + +Transformers `generate()` 的常规 sampling 最终调用 `torch.multinomial`。即使四行共享 batch +seed，每行也消费不同 RNG 子流；如果逐行重置同 seed，分布内部的实现细节仍不等于我们 +明确定义并能逐步审计的共同随机数合同。 + +本轮不调用 `torch.multinomial`，也不把输出称作“Transformers 官方 sampler 的 exact +trajectory”。模型、logits 和官方 `.3/.95` 解码分布仍来自固定 checkpoint；离散取样器改为 +本协议定义的**显式均匀数 + token-ID 顺序逆 CDF**。 + +### 4.2 冻结随机带 + +四条带 T0–T3 的 display seed： + +| 带 | display seed | 派生字符串 SHA-256 | +|---|---:|---| +| T0 | 2,572,353,518 | `9952ffeec840649d2654d35b9a86c97db3737b70826c366e3f5a7633ec26d0c4` | +| T1 | 399,507,326 | `17cfff7e5b3b6d011f4d1b8076637d4bdfb0ebd8a6021beb2a0011a8f6d520c6` | +| T2 | 405,284,229 | `1828258525ea1b450d6de0ba0d9ff12f737b048b1cef554d7dd2ba90d25e5f4d` | +| T3 | 568,701,915 | `21e5b3dbe506f1b7e9d483318195fc5cb4990c518850a52d40b5eff225491bd2` | + +对 `tape_label, source_id, step`： + +```text +H = SHA256( + protocol_id + "\0uniform\0" + + tape_label + "\0" + + source_id + "\0" + + decimal(step) +) + +z_t = H[0:8]，big-endian unsigned uint64 +u_t = (z_t + 0.5) / 2^64 +``` + +`u_t` 位于开区间 `(0,1)`。同一 source + tape 的四个条件在相同生成步 `t` 使用完全相同 +的 `z_t`。某条件 EOS 后停止，不再产生“配对输出”；其他仍存活条件继续读取下一步相同 +定义的随机带。 + +### 4.3 每一步的概率与取样 + +对模型最后位置 logits： + +1. 用 Transformers `TemperatureLogitsWarper(0.3)`； +2. 用 Transformers `TopPLogitsWarper(0.95, min_tokens_to_keep=1)`； +3. 以 `torch.float32` 做 softmax； +4. 按 token ID `0..vocab-1` 做 `torch.float32` cumulative sum； +5. 强制最后一个 CDF 元素为 `1.0`； +6. 将 `u_t` cast 为 `torch.float32`； +7. `torch.searchsorted(cdf, u_t, right=False)` 取得 token ID。 + +因此： + +```text +共享的是概率积分尺度上的 u_t +≠ 强迫四格选择同一 token +``` + +prompt 改变 logits 后，同一个 `u_t` 可以落入不同 token 区间，这正是共同随机数配对所需 +的性质。 + +每条 output 记录： + +- 前 8 个 `z_t` / `u_t`； +-实际消费的 `z_t` 序列规范 hash； +- prompt hash、generated token IDs 与 hash； +- EOS / truncation； +-生成前后 CPU/CUDA RNG state hash； +- PyTorch RNG 是否完全未被取样器消费。 + +--- + +## 5. 执行网格、smoke 与独立重放 + +### 5.1 16-token smoke + +每域 index `0,8,16,24`，T0/T1，四条件： + +```text +8 sources × 2 tapes × 4 conditions = 64 short outputs +``` + +通过闸门： + +1. 正式 64 题 × 4 条件的 256 个 prompt hash 全部与清单 exact； +2. 64 条短输出无 OOM、NaN、exception； +3. 同进程重放 8 sources × T0 × 4 = 32 cells 逐 token、文本与 stop state exact； +4. T0/T1 至少一个 source-condition trajectory 分叉； +5. 同一 source+tape 四条件的前 16 个 `z_t` 定义逐步 exact； +6. 生成前后 PyTorch CPU/CUDA RNG state exact，证明 sampling 不消费隐式 RNG； +7. synthetic distribution 单元测试覆盖 CDF 边界与被 top-p 过滤 token。 + +smoke 不进入正式统计。 + +### 5.2 正式运行 + +- 全部 64 题跑 T0 × 四条件：256 outputs； +- 每域 index `0,8,16,24` 再跑 T1/T2/T3 × 四条件：96 outputs； +- 合计 352 outputs； +- 每题结束后写独立 journal；再次启动时必须核验协议、source、prompt 与输出内容 hash 后才 + 能 resume； +- journal 只用于可恢复执行，正式 JSON 由全部合格 journal 组装。 + +### 5.3 跨进程独立重放 + +每域 index： + +```text +0, 4, 8, 12, 16, 20, 24, 28 +``` + +只跑 T0 × 四条件： + +```text +16 sources × 1 tape × 4 conditions = 64 outputs +``` + +逐格比较 protocol、model revision、generation contract、tape、prompt hash、uniform hash、 +generated token IDs、decoded text、EOS、truncation 和生成前 RNG state。目标为 64/64 +全部 exact。 + +--- + +## 6. evaluator 在输出前冻结 + +### 6.1 HumanEval + +沿用前序四本账： + +1. 代码抽取是否成功； +2. Python AST 是否可解析； +3. sandbox 是否真正执行； +4. tests 是否通过。 + +固定 sandbox： + +```text +python:3.11-alpine +@sha256:25976e9d34a0fab1f278cae931f34c8303d97bf0c0d7f85b6b4dcf641d7702a4 +``` + +仍为 network none、read-only root filesystem、非 root、drop ALL capabilities、 +no-new-privileges、256 MiB、64 PIDs、0.5 CPU、5 秒 timeout。报告失败类型： + +```text +extract_failed / syntax_error / timeout / runtime_error / assertion_failed / passed +``` + +主 correctness 是 fixed-budget tests pass；strict-complete pass 单独报告，不混写。 + +### 6.2 GSM8K + +沿用冻结 numeric final-answer evaluator： + +- gold：`####` 后最后一个规范化数字； +- prediction：输出中最后一个规范化数字； +- fixed-budget numeric exact； +- strict-complete numeric exact； +- 无数字单独记为 `no_numeric_answer`； +- 有数字但不等于 gold 记为 `wrong_numeric_answer`。 + +--- + +## 7. 统计合同 + +### 7.1 主要 contrast + +在每个 domain 内、每道题、T0 上计算右减左： + +```text +period_at_s0 = s0_period - s0_eos +period_at_s1 = s1_period - s1_eos +system_at_eos = s1_eos - s0_eos +system_at_period = s1_period - s0_period +``` + +success 用 `{-1,0,+1}` 任务级转移；长度与 EOS 保留配对差。两因子 main effect 与 +interaction 可以作为 secondary descriptive summary，但不能替代四个预注册 contrast。 + +### 7.2 任务 bootstrap + +HumanEval 32 题与 GSM8K 32 题分别进行： + +- 10,000 次； +- 每次在当前 domain 的 32 题中有放回抽 32 题； +- 同一次 resample 对四条件保持题级配对； +- 固定 seed `1,364,512,825`； +- 报告 percentile 2.5% / 50% / 97.5%。 + +必须写成： + +> 当前冻结 32 题、固定 T0 下的 selected-task resampling band。 + +不得写成： + +- 完整 benchmark 总体 95% CI； +- 模型真实能力 CI； +- 同时覆盖 generation-seed uncertainty 的 CI； +- 因果效应 CI。 + +### 7.3 多随机带诊断 + +对每域固定四题 × T0–T3： + +- 分别报告每条带的 condition success / EOS / length； +- 计算题内跨带 range 与带内跨题 range； +- 分别做 selected-task 和 selected-tape 描述； +- 不把 `4 tasks × 4 tapes = 16` 当成 16 道独立题。 + +--- + +## 8. 结论边界 + +允许： + +- 描述这 32 道预选题、T0 下四个条件的配对差； +- 描述任务重采样带与 8 道诊断题上的 tape sensitivity； +- 描述显式共同随机数下的 first-divergence、转移表和长度差； +- 报告 exact replay 是否通过。 + +禁止： + +- 把 period 当成官方可部署格式； +- 把 selected-task band 外推为 benchmark / 用户分布总体； +- 把一条 tape 的结果写成 sampling robustness； +- 把测试通过写成代码安全； +- 把同一 `u_t` 错写成“同一 token”； +- 把本协议 sampler 写成 Transformers `generate()` / `torch.multinomial` 的 exact + trajectory； +- 根据正式输出修改 source、tape、contrast、bootstrap seed、sandbox 或主指标。 + +任何偏离必须在结果审计中逐项列出；不能静默修订本文件。 diff --git a/scripts/build-deepseek-chat-task-bootstrap-manifest.mjs b/scripts/build-deepseek-chat-task-bootstrap-manifest.mjs new file mode 100644 index 0000000..509a2b0 --- /dev/null +++ b/scripts/build-deepseek-chat-task-bootstrap-manifest.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import process from "node:process"; + +const PROTOCOL_ID = "llm-atlas-deepseek-chat-task-bootstrap-crn-v1"; +const CONDITIONS = ["s0_eos", "s1_eos", "s0_period", "s1_period"]; +const DOMAINS = ["code", "math"]; +const DIAGNOSTIC_INDICES = new Set([0, 8, 16, 24]); +const REPLAY_INDICES = new Set([0, 4, 8, 12, 16, 20, 24, 28]); + +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function uint32(label) { + return createHash("sha256").update(label).digest().readUInt32BE(0); +} + +function canonical(value) { + if (Array.isArray(value)) { + return `[${value.map(canonical).join(",")}]`; + } + if (value !== null && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function parseArgs(argv) { + const args = {}; + for (let index = 2; index < argv.length; index += 2) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith("--") || value === undefined) { + throw new Error(`Expected --name value arguments; got ${key ?? ""}`); + } + args[key.slice(2)] = value; + } + if (!args.reference || !args.output) { + throw new Error("Usage: build-deepseek-chat-task-bootstrap-manifest.mjs --reference FILE --output FILE"); + } + return args; +} + +const args = parseArgs(process.argv); +const referencePath = resolve(args.reference); +const outputPath = resolve(args.output); +const referenceBytes = await readFile(referencePath); +const reference = JSON.parse(referenceBytes); +const selected = reference.corpus_contract?.selected ?? []; + +const sources = DOMAINS.flatMap((domain) => { + const rows = selected + .filter((row) => row.domain === domain) + .sort((left, right) => left.within_domain_index - right.within_domain_index); + if (rows.length !== 32) { + throw new Error(`${domain}: expected 32 frozen sources, observed ${rows.length}`); + } + return rows.map((row, domainIndex) => { + if (row.within_domain_index !== domainIndex) { + throw new Error(`${domain}: non-contiguous within_domain_index at ${domainIndex}`); + } + const promptHashes = Object.fromEntries( + CONDITIONS.map((condition) => { + const hash = row.conditions?.[condition]?.token_ids_sha256; + if (!/^[0-9a-f]{64}$/.test(hash ?? "")) { + throw new Error(`${row.id}/${condition}: missing prompt hash`); + } + return [condition, hash]; + }), + ); + return { + id: row.id, + domain, + domain_index: domainIndex, + selection_rank: row.selection_rank, + source_text_sha256: row.text_sha256, + prompt_token_ids_sha256: promptHashes, + main_tapes: ["T0"], + diagnostic_tapes: DIAGNOSTIC_INDICES.has(domainIndex) + ? ["T1", "T2", "T3"] + : [], + independent_replay: REPLAY_INDICES.has(domainIndex), + }; + }); +}); + +const tapes = Array.from({ length: 4 }, (_, index) => { + const label = `T${index}`; + const derivation = `${PROTOCOL_ID}/tape/${index}`; + return { + label, + index, + display_seed: uint32(derivation), + derivation, + derivation_sha256: sha256(derivation), + }; +}); + +const manifest = { + schema_version: 1, + protocol_id: PROTOCOL_ID, + status: "preregistered_before_any_protocol_output", + source_reference: { + path: args.reference, + sha256: sha256(referenceBytes), + sample_salt: reference.corpus_contract.sample_salt, + selection_rule: + "For each domain, all 32 rows ordered by frozen within_domain_index; no outcome-based selection.", + }, + model: { + repo: "deepseek-ai/DeepSeek-V2-Lite-Chat", + revision: "85864749cd611b4353ce1decdb286193298f64c7", + }, + conditions: CONDITIONS, + domains: DOMAINS, + sources, + source_counts: Object.fromEntries( + DOMAINS.map((domain) => [ + domain, + sources.filter((source) => source.domain === domain).length, + ]), + ), + tape_contract: { + tapes, + main_tape: "T0", + diagnostic_source_indices_per_domain: [...DIAGNOSTIC_INDICES], + diagnostic_tapes: ["T0", "T1", "T2", "T3"], + uniform_uint64: + "first 8 SHA-256 bytes, big-endian, of protocol_id + '\\0uniform\\0' + tape_label + '\\0' + source_id + '\\0' + decimal(step)", + uniform_open_interval: + "u_t = (uniform_uint64 + 0.5) / 2^64; cast to torch.float32 before searchsorted", + common_random_number_scope: + "For the same source and tape, all four conditions consume the identical u_t at generation step t while each trajectory remains active.", + }, + execution_grid: { + main: { + sources: 64, + tapes_per_source: 1, + conditions: 4, + outputs: 256, + }, + additional_tape_diagnostic: { + sources: 8, + additional_tapes_per_source: 3, + conditions: 4, + outputs: 96, + }, + formal_outputs: 352, + independent_replay: { + source_indices_per_domain: [...REPLAY_INDICES], + tapes: ["T0"], + conditions: 4, + outputs: 64, + }, + }, + bootstrap_contract: { + resamples: 10000, + domain_separated: true, + main_task_bootstrap_seed: uint32( + `${PROTOCOL_ID}/bootstrap/task-bootstrap`, + ), + diagnostic_task_bootstrap_seed: uint32( + `${PROTOCOL_ID}/bootstrap/diagnostic-task-bootstrap`, + ), + tape_bootstrap_seed: uint32( + `${PROTOCOL_ID}/bootstrap/tape-bootstrap`, + ), + estimand_boundary: + "Selected-task resampling bands for this fixed 32-task frame and one frozen tape; not benchmark-population or generation-seed confidence intervals.", + }, +}; + +manifest.canonical_content_sha256 = sha256(canonical(manifest)); +await writeFile(outputPath, `${JSON.stringify(manifest, null, 2)}\n`); + +const outputBytes = await readFile(outputPath); +process.stdout.write( + `${JSON.stringify( + { + output: outputPath, + bytes: outputBytes.length, + sha256: sha256(outputBytes), + canonical_content_sha256: manifest.canonical_content_sha256, + sources: sources.length, + formal_outputs: manifest.execution_grid.formal_outputs, + replay_outputs: manifest.execution_grid.independent_replay.outputs, + }, + null, + 2, + )}\n`, +);