{ "schemaVersion": 1, "protocolId": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1", "capturedAt": "2026-07-29T20:25:26.335344+00:00", "model": { "repo": "deepseek-ai/DeepSeek-V2-Lite-Chat", "revision": "85864749cd611b4353ce1decdb286193298f64c7", "checkpointIdentity": "SFT Chat", "dtype": "bfloat16" }, "artifactHashes": { "sampling": "ea0607f2b197fac3f794655c1538ce1f9a1cb072d637eb31d35573682e311809", "evaluation": "82b2fc5d1f854a7e0cd7aba7c70ff74d733d24221522a4f92b962478c76177ab", "reproduction": "63ed39e5dcdbc2a30e516172f3657dfa453ff3b23d5bd5c49c734939242a70f5", "analysis": "9ab17561ced930a668c082141f7c6e013cbda70e42b09de63d41f1b82c01a6ae" }, "grid": { "formalSources": 64, "formalRuns": 88, "formalOutputs": 352, "mainT0Outputs": 256, "diagnosticAdditionalOutputs": 96, "naturalEos": 343, "budgetTruncated": 9, "uniqueTrajectories": 320, "promptHashesExact": 256, "torchRngUnchangedRuns": 88 }, "sampler": { "name": "explicit-uniform inverse-CDF nucleus sampler in token-ID order", "temperature": 0.3, "topP": 0.95, "softmaxDtype": "torch.float32", "cdfDtype": "torch.float32", "uniformDtype": "torch.float32", "transformersGenerateCalled": false, "torchMultinomialCalled": false, "commonRandomNumbers": true }, "conditionTable": { "code": [ { "condition": "s0_eos", "outputs": 32, "success": 16, "successRate": 0.5, "naturalEos": 30, "naturalEosRate": 0.9375, "meanTokens": 330.09375, "outcomes": { "assertion_failed": 10, "extract_failed": 1, "passed": 16, "runtime_error": 4, "timeout": 1 } }, { "condition": "s1_eos", "outputs": 32, "success": 16, "successRate": 0.5, "naturalEos": 30, "naturalEosRate": 0.9375, "meanTokens": 308.9375, "outcomes": { "assertion_failed": 11, "passed": 16, "runtime_error": 4, "timeout": 1 } }, { "condition": "s0_period", "outputs": 32, "success": 15, "successRate": 0.46875, "naturalEos": 31, "naturalEosRate": 0.96875, "meanTokens": 314.375, "outcomes": { "assertion_failed": 12, "passed": 15, "runtime_error": 3, "syntax_error": 1, "timeout": 1 } }, { "condition": "s1_period", "outputs": 32, "success": 12, "successRate": 0.375, "naturalEos": 31, "naturalEosRate": 0.96875, "meanTokens": 183.5, "outcomes": { "assertion_failed": 14, "extract_failed": 1, "passed": 12, "runtime_error": 5 } } ], "math": [ { "condition": "s0_eos", "outputs": 32, "success": 19, "successRate": 0.59375, "naturalEos": 31, "naturalEosRate": 0.96875, "meanTokens": 181.40625, "outcomes": { "passed": 19, "wrong_numeric_answer": 13 } }, { "condition": "s1_eos", "outputs": 32, "success": 19, "successRate": 0.59375, "naturalEos": 32, "naturalEosRate": 1, "meanTokens": 190.96875, "outcomes": { "passed": 19, "wrong_numeric_answer": 13 } }, { "condition": "s0_period", "outputs": 32, "success": 17, "successRate": 0.53125, "naturalEos": 32, "naturalEosRate": 1, "meanTokens": 173.1875, "outcomes": { "passed": 17, "wrong_numeric_answer": 15 } }, { "condition": "s1_period", "outputs": 32, "success": 16, "successRate": 0.5, "naturalEos": 32, "naturalEosRate": 1, "meanTokens": 198.3125, "outcomes": { "passed": 16, "wrong_numeric_answer": 16 } } ] }, "contrasts": { "code": { "period_at_s0": { "left": "s0_eos", "right": "s0_period", "metrics": { "fixed_budget_success": { "point": -0.03125, "band": { "p2_5": -0.15625, "p50": -0.03125, "p97_5": 0.09375 }, "directions": { "positive": 2, "zero": 27, "negative": 3 }, "transition": { "fail_to_fail": 14, "fail_to_pass": 2, "pass_to_fail": 3, "pass_to_pass": 13 } }, "strict_complete_success": { "point": -0.03125, "band": { "p2_5": -0.15625, "p50": -0.03125, "p97_5": 0.09375 }, "directions": { "positive": 2, "zero": 27, "negative": 3 }, "transition": { "fail_to_fail": 14, "fail_to_pass": 2, "pass_to_fail": 3, "pass_to_pass": 13 } }, "natural_eos": { "point": 0.03125, "band": { "p2_5": -0.0625, "p50": 0.03125, "p97_5": 0.15625 }, "directions": { "positive": 2, "zero": 29, "negative": 1 }, "transition": null }, "generated_tokens": { "point": -15.71875, "band": { "p2_5": -57.627343749999994, "p50": -15.34375, "p97_5": 24.93828124999999 }, "directions": { "positive": 15, "zero": 0, "negative": 17 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 0, "commonPrefixTokens": { "count": 32, "mean": 13.25, "median": 10, "min": 0, "max": 65 } } }, "period_at_s1": { "left": "s1_eos", "right": "s1_period", "metrics": { "fixed_budget_success": { "point": -0.125, "band": { "p2_5": -0.34375, "p50": -0.125, "p97_5": 0.09375 }, "directions": { "positive": 4, "zero": 20, "negative": 8 }, "transition": { "fail_to_fail": 12, "fail_to_pass": 4, "pass_to_fail": 8, "pass_to_pass": 8 } }, "strict_complete_success": { "point": -0.125, "band": { "p2_5": -0.34375, "p50": -0.125, "p97_5": 0.09375 }, "directions": { "positive": 4, "zero": 20, "negative": 8 }, "transition": { "fail_to_fail": 12, "fail_to_pass": 4, "pass_to_fail": 8, "pass_to_pass": 8 } }, "natural_eos": { "point": 0.03125, "band": { "p2_5": -0.0625, "p50": 0.03125, "p97_5": 0.125 }, "directions": { "positive": 2, "zero": 29, "negative": 1 }, "transition": null }, "generated_tokens": { "point": -125.4375, "band": { "p2_5": -165.03125, "p50": -125.71875, "p97_5": -85.125 }, "directions": { "positive": 4, "zero": 0, "negative": 28 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 0, "commonPrefixTokens": { "count": 32, "mean": 3.6875, "median": 0, "min": 0, "max": 32 } } }, "system_at_eos": { "left": "s0_eos", "right": "s1_eos", "metrics": { "fixed_budget_success": { "point": 0, "band": { "p2_5": -0.125, "p50": 0, "p97_5": 0.125 }, "directions": { "positive": 2, "zero": 28, "negative": 2 }, "transition": { "fail_to_fail": 14, "fail_to_pass": 2, "pass_to_fail": 2, "pass_to_pass": 14 } }, "strict_complete_success": { "point": 0, "band": { "p2_5": -0.125, "p50": 0, "p97_5": 0.125 }, "directions": { "positive": 2, "zero": 28, "negative": 2 }, "transition": { "fail_to_fail": 14, "fail_to_pass": 2, "pass_to_fail": 2, "pass_to_pass": 14 } }, "natural_eos": { "point": 0, "band": { "p2_5": 0, "p50": 0, "p97_5": 0 }, "directions": { "positive": 0, "zero": 32, "negative": 0 }, "transition": null }, "generated_tokens": { "point": -21.15625, "band": { "p2_5": -47.846093749999994, "p50": -20.78125, "p97_5": 2.4382812499999886 }, "directions": { "positive": 11, "zero": 2, "negative": 19 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 0, "commonPrefixTokens": { "count": 32, "mean": 23.90625, "median": 15, "min": 0, "max": 107 } } }, "system_at_period": { "left": "s0_period", "right": "s1_period", "metrics": { "fixed_budget_success": { "point": -0.09375, "band": { "p2_5": -0.25, "p50": -0.09375, "p97_5": 0.0625 }, "directions": { "positive": 2, "zero": 25, "negative": 5 }, "transition": { "fail_to_fail": 15, "fail_to_pass": 2, "pass_to_fail": 5, "pass_to_pass": 10 } }, "strict_complete_success": { "point": -0.09375, "band": { "p2_5": -0.25, "p50": -0.09375, "p97_5": 0.0625 }, "directions": { "positive": 2, "zero": 25, "negative": 5 }, "transition": { "fail_to_fail": 15, "fail_to_pass": 2, "pass_to_fail": 5, "pass_to_pass": 10 } }, "natural_eos": { "point": 0, "band": { "p2_5": -0.09375, "p50": 0, "p97_5": 0.09375 }, "directions": { "positive": 1, "zero": 30, "negative": 1 }, "transition": null }, "generated_tokens": { "point": -130.875, "band": { "p2_5": -178.6875, "p50": -131.28125, "p97_5": -83.21796875000001 }, "directions": { "positive": 6, "zero": 1, "negative": 25 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 1, "commonPrefixTokens": { "count": 32, "mean": 11.875, "median": 0, "min": 0, "max": 109 } } } }, "math": { "period_at_s0": { "left": "s0_eos", "right": "s0_period", "metrics": { "fixed_budget_success": { "point": -0.0625, "band": { "p2_5": -0.21875, "p50": -0.0625, "p97_5": 0.09375 }, "directions": { "positive": 3, "zero": 24, "negative": 5 }, "transition": { "fail_to_fail": 10, "fail_to_pass": 3, "pass_to_fail": 5, "pass_to_pass": 14 } }, "strict_complete_success": { "point": -0.0625, "band": { "p2_5": -0.21875, "p50": -0.0625, "p97_5": 0.09375 }, "directions": { "positive": 3, "zero": 24, "negative": 5 }, "transition": { "fail_to_fail": 10, "fail_to_pass": 3, "pass_to_fail": 5, "pass_to_pass": 14 } }, "natural_eos": { "point": 0.03125, "band": { "p2_5": 0, "p50": 0.03125, "p97_5": 0.09375 }, "directions": { "positive": 1, "zero": 31, "negative": 0 }, "transition": null }, "generated_tokens": { "point": -8.21875, "band": { "p2_5": -27.56328125, "p50": -7.8125, "p97_5": 8.6875 }, "directions": { "positive": 10, "zero": 10, "negative": 12 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 6, "commonPrefixTokens": { "count": 32, "mean": 52.53125, "median": 43, "min": 0, "max": 254 } } }, "period_at_s1": { "left": "s1_eos", "right": "s1_period", "metrics": { "fixed_budget_success": { "point": -0.09375, "band": { "p2_5": -0.25, "p50": -0.09375, "p97_5": 0.0625 }, "directions": { "positive": 2, "zero": 25, "negative": 5 }, "transition": { "fail_to_fail": 11, "fail_to_pass": 2, "pass_to_fail": 5, "pass_to_pass": 14 } }, "strict_complete_success": { "point": -0.09375, "band": { "p2_5": -0.25, "p50": -0.09375, "p97_5": 0.0625 }, "directions": { "positive": 2, "zero": 25, "negative": 5 }, "transition": { "fail_to_fail": 11, "fail_to_pass": 2, "pass_to_fail": 5, "pass_to_pass": 14 } }, "natural_eos": { "point": 0, "band": { "p2_5": 0, "p50": 0, "p97_5": 0 }, "directions": { "positive": 0, "zero": 32, "negative": 0 }, "transition": null }, "generated_tokens": { "point": 7.34375, "band": { "p2_5": -9.40703125, "p50": 7.140625, "p97_5": 26.09375 }, "directions": { "positive": 15, "zero": 1, "negative": 16 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 1, "commonPrefixTokens": { "count": 32, "mean": 22.90625, "median": 8.5, "min": 0, "max": 152 } } }, "system_at_eos": { "left": "s0_eos", "right": "s1_eos", "metrics": { "fixed_budget_success": { "point": 0, "band": { "p2_5": -0.15625, "p50": 0, "p97_5": 0.15625 }, "directions": { "positive": 3, "zero": 26, "negative": 3 }, "transition": { "fail_to_fail": 10, "fail_to_pass": 3, "pass_to_fail": 3, "pass_to_pass": 16 } }, "strict_complete_success": { "point": 0, "band": { "p2_5": -0.15625, "p50": 0, "p97_5": 0.15625 }, "directions": { "positive": 3, "zero": 26, "negative": 3 }, "transition": { "fail_to_fail": 10, "fail_to_pass": 3, "pass_to_fail": 3, "pass_to_pass": 16 } }, "natural_eos": { "point": 0.03125, "band": { "p2_5": 0, "p50": 0.03125, "p97_5": 0.09375 }, "directions": { "positive": 1, "zero": 31, "negative": 0 }, "transition": null }, "generated_tokens": { "point": 9.5625, "band": { "p2_5": -4.9375, "p50": 9.234375, "p97_5": 25.1875 }, "directions": { "positive": 15, "zero": 8, "negative": 9 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 6, "commonPrefixTokens": { "count": 32, "mean": 59.4375, "median": 44.5, "min": 0, "max": 213 } } }, "system_at_period": { "left": "s0_period", "right": "s1_period", "metrics": { "fixed_budget_success": { "point": -0.03125, "band": { "p2_5": -0.15625, "p50": -0.03125, "p97_5": 0.09375 }, "directions": { "positive": 2, "zero": 27, "negative": 3 }, "transition": { "fail_to_fail": 13, "fail_to_pass": 2, "pass_to_fail": 3, "pass_to_pass": 14 } }, "strict_complete_success": { "point": -0.03125, "band": { "p2_5": -0.15625, "p50": -0.03125, "p97_5": 0.09375 }, "directions": { "positive": 2, "zero": 27, "negative": 3 }, "transition": { "fail_to_fail": 13, "fail_to_pass": 2, "pass_to_fail": 3, "pass_to_pass": 14 } }, "natural_eos": { "point": 0, "band": { "p2_5": 0, "p50": 0, "p97_5": 0 }, "directions": { "positive": 0, "zero": 32, "negative": 0 }, "transition": null }, "generated_tokens": { "point": 25.125, "band": { "p2_5": 7.874218750000001, "p50": 24.84375, "p97_5": 44.53125 }, "directions": { "positive": 18, "zero": 2, "negative": 12 }, "transition": null } }, "trajectory": { "sources": 32, "sharedUniformExact": 32, "exactTrajectories": 1, "commonPrefixTokens": { "count": 32, "mean": 27.75, "median": 9.5, "min": 0, "max": 152 } } } } }, "diagnostic": { "code": { "sourceIds": [ "HumanEval/31", "HumanEval/22", "HumanEval/53", "HumanEval/141" ], "tapes": [ "T0", "T1", "T2", "T3" ], "contrasts": { "period_at_s0": { "left": "s0_eos", "right": "s0_period", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ] ], "tapeMeans": { "T0": 0, "T1": 0, "T2": 0, "T3": 0 }, "taskRangeWithinTape": { "count": 4, "mean": 0, "median": 0, "min": 0, "max": 0 }, "tapeRangeWithinTask": { "count": 4, "mean": 0, "median": 0, "min": 0, "max": 0 } }, "tokens": { "matrix": [ [ 26, 59, 13, -51 ], [ 49, -7, -38, 49 ], [ -48, 42, -58, -41 ], [ -85, 84, 137, -24 ] ], "tapeMeans": { "T0": -14.5, "T1": 44.5, "T2": 13.5, "T3": -16.75 }, "taskRangeWithinTape": { "count": 4, "mean": 130, "median": 117, "min": 91, "max": 195 }, "tapeRangeWithinTask": { "count": 4, "mean": 129.75, "median": 105, "min": 87, "max": 222 } } }, "period_at_s1": { "left": "s1_eos", "right": "s1_period", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ], [ -1, 0, 0, 0 ], [ 0, 0, 0, 0 ] ], "tapeMeans": { "T0": -0.25, "T1": 0, "T2": 0, "T3": 0 }, "taskRangeWithinTape": { "count": 4, "mean": 0.25, "median": 0, "min": 0, "max": 1 }, "tapeRangeWithinTask": { "count": 4, "mean": 0.25, "median": 0, "min": 0, "max": 1 } }, "tokens": { "matrix": [ [ -285, -249, -222, -266 ], [ 22, -35, 120, -39 ], [ -121, -70, -5, -100 ], [ -50, -266, 23, -248 ] ], "tapeMeans": { "T0": -108.5, "T1": -155, "T2": -21, "T3": -163.25 }, "taskRangeWithinTape": { "count": 4, "mean": 276.75, "median": 269, "min": 227, "max": 342 }, "tapeRangeWithinTask": { "count": 4, "mean": 156.75, "median": 137.5, "min": 63, "max": 289 } } }, "system_at_eos": { "left": "s0_eos", "right": "s1_eos", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ] ], "tapeMeans": { "T0": 0, "T1": 0, "T2": 0, "T3": 0 }, "taskRangeWithinTape": { "count": 4, "mean": 0, "median": 0, "min": 0, "max": 0 }, "tapeRangeWithinTask": { "count": 4, "mean": 0, "median": 0, "min": 0, "max": 0 } }, "tokens": { "matrix": [ [ -50, -64, -77, -113 ], [ 26, 13, -154, 22 ], [ -6, -9, -58, -48 ], [ -206, 84, -71, -19 ] ], "tapeMeans": { "T0": -59, "T1": 6, "T2": -90, "T3": -39.5 }, "taskRangeWithinTape": { "count": 4, "mean": 152.75, "median": 141.5, "min": 96, "max": 232 }, "tapeRangeWithinTask": { "count": 4, "mean": 146.25, "median": 121.5, "min": 52, "max": 290 } } }, "system_at_period": { "left": "s0_period", "right": "s1_period", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ 0, 0, 0, 0 ], [ -1, 0, 0, 0 ], [ 0, 0, 0, 0 ] ], "tapeMeans": { "T0": -0.25, "T1": 0, "T2": 0, "T3": 0 }, "taskRangeWithinTape": { "count": 4, "mean": 0.25, "median": 0, "min": 0, "max": 1 }, "tapeRangeWithinTask": { "count": 4, "mean": 0.25, "median": 0, "min": 0, "max": 1 } }, "tokens": { "matrix": [ [ -361, -372, -312, -328 ], [ -1, -15, 4, -66 ], [ -79, -121, -5, -107 ], [ -171, -266, -185, -243 ] ], "tapeMeans": { "T0": -153, "T1": -193.5, "T2": -124.5, "T3": -186 }, "taskRangeWithinTape": { "count": 4, "mean": 323.75, "median": 336.5, "min": 262, "max": 360 }, "tapeRangeWithinTask": { "count": 4, "mean": 85.25, "median": 82.5, "min": 60, "max": 116 } } } } }, "math": { "sourceIds": [ "gsm8k/test/1069", "gsm8k/test/0546", "gsm8k/test/1050", "gsm8k/test/0584" ], "tapes": [ "T0", "T1", "T2", "T3" ], "contrasts": { "period_at_s0": { "left": "s0_eos", "right": "s0_period", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ -1, 0, 0, 1 ], [ 0, 0, 0, 0 ], [ 1, 0, -1, 0 ] ], "tapeMeans": { "T0": 0, "T1": 0, "T2": -0.25, "T3": 0.25 }, "taskRangeWithinTape": { "count": 4, "mean": 1, "median": 1, "min": 0, "max": 2 }, "tapeRangeWithinTask": { "count": 4, "mean": 1, "median": 1, "min": 0, "max": 2 } }, "tokens": { "matrix": [ [ 0, 48, -19, 0 ], [ -13, -9, 0, 8 ], [ 0, 35, -1, 0 ], [ -8, 51, -20, 8 ] ], "tapeMeans": { "T0": -5.25, "T1": 31.25, "T2": -10, "T3": 4 }, "taskRangeWithinTape": { "count": 4, "mean": 25.25, "median": 16.5, "min": 8, "max": 60 }, "tapeRangeWithinTask": { "count": 4, "mean": 48.75, "median": 51.5, "min": 21, "max": 71 } } }, "period_at_s1": { "left": "s1_eos", "right": "s1_period", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ 0, -1, 0, -1 ], [ -1, -1, -1, -1 ], [ -1, -1, 0, 0 ] ], "tapeMeans": { "T0": -0.5, "T1": -0.75, "T2": -0.25, "T3": -0.5 }, "taskRangeWithinTape": { "count": 4, "mean": 1, "median": 1, "min": 1, "max": 1 }, "tapeRangeWithinTask": { "count": 4, "mean": 0.5, "median": 0.5, "min": 0, "max": 1 } }, "tokens": { "matrix": [ [ 16, -6, 30, -17 ], [ 13, -25, -27, -44 ], [ 3, 67, 40, 106 ], [ -30, -35, -30, -55 ] ], "tapeMeans": { "T0": 0.5, "T1": 0.25, "T2": 3.25, "T3": -2.5 }, "taskRangeWithinTape": { "count": 4, "mean": 94.75, "median": 86, "min": 46, "max": 161 }, "tapeRangeWithinTask": { "count": 4, "mean": 58, "median": 52, "min": 25, "max": 103 } } }, "system_at_eos": { "left": "s0_eos", "right": "s1_eos", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ -1, 1, 0, 1 ], [ 0, 0, 0, 0 ], [ 1, 1, -1, 0 ] ], "tapeMeans": { "T0": 0, "T1": 0.5, "T2": -0.25, "T3": 0.25 }, "taskRangeWithinTape": { "count": 4, "mean": 1.25, "median": 1, "min": 1, "max": 2 }, "tapeRangeWithinTask": { "count": 4, "mean": 1, "median": 1, "min": 0, "max": 2 } }, "tokens": { "matrix": [ [ -36, 64, -17, -3 ], [ -26, 24, 0, 8 ], [ 34, 37, -1, 0 ], [ 16, 94, -12, 126 ] ], "tapeMeans": { "T0": -3, "T1": 54.75, "T2": -7.5, "T3": 32.75 }, "taskRangeWithinTape": { "count": 4, "mean": 71.5, "median": 70, "min": 17, "max": 129 }, "tapeRangeWithinTask": { "count": 4, "mean": 81.5, "median": 75, "min": 38, "max": 138 } } }, "system_at_period": { "left": "s0_period", "right": "s1_period", "success": { "matrix": [ [ 0, 0, 0, 0 ], [ 0, 0, 0, -1 ], [ -1, -1, -1, -1 ], [ -1, 0, 0, 0 ] ], "tapeMeans": { "T0": -0.5, "T1": -0.25, "T2": -0.25, "T3": -0.5 }, "taskRangeWithinTape": { "count": 4, "mean": 1, "median": 1, "min": 1, "max": 1 }, "tapeRangeWithinTask": { "count": 4, "mean": 0.5, "median": 0.5, "min": 0, "max": 1 } }, "tokens": { "matrix": [ [ -20, 10, 32, -20 ], [ 0, 8, -27, -44 ], [ 37, 69, 40, 106 ], [ -6, 8, -22, 63 ] ], "tapeMeans": { "T0": 2.75, "T1": 23.75, "T2": 5.75, "T3": 26.25 }, "taskRangeWithinTape": { "count": 4, "mean": 83.75, "median": 64, "min": 57, "max": 150 }, "tapeRangeWithinTask": { "count": 4, "mean": 64.5, "median": 60.5, "min": 52, "max": 85 } } } } } }, "tasks": { "code": [ { "id": "HumanEval/31", "index": 0, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 394, "naturalEos": true, "truncated": false, "trajectoryHash": "9bf7b1d42c68", "firstTokenIds": [ 2158, 4353, 254, 2030, 262, 62, 6620, 63 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 344, "naturalEos": true, "truncated": false, "trajectoryHash": "b2c8363abfa6", "firstTokenIds": [ 2158, 6983, 565, 245, 1604, 317, 9966, 11 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 420, "naturalEos": true, "truncated": false, "trajectoryHash": "4983d736318e", "firstTokenIds": [ 2158, 4353, 254, 2030, 262, 62, 6620, 63 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 59, "naturalEos": true, "truncated": false, "trajectoryHash": "5cc9c902a87c", "firstTokenIds": [ 977, 317, 62, 6620, 7, 77, 1780, 185 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 26, "commonPrefixTokens": 27, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -285, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -50, "commonPrefixTokens": 1, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -361, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "HumanEval/44", "index": 1, "conditions": { "s0_eos": { "success": false, "outcome": "runtime_error", "tokens": 496, "naturalEos": true, "truncated": false, "trajectoryHash": "19e1162b6f49", "firstTokenIds": [ 2158, 4353, 254, 2030, 4874, 62, 4915, 63 ] }, "s1_eos": { "success": false, "outcome": "runtime_error", "tokens": 445, "naturalEos": true, "truncated": false, "trajectoryHash": "5f0d89d2123c", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 543, 2606 ] }, "s0_period": { "success": false, "outcome": "runtime_error", "tokens": 498, "naturalEos": true, "truncated": false, "trajectoryHash": "e23ca257f456", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 543, 938 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 180, "naturalEos": true, "truncated": false, "trajectoryHash": "6b5783005111", "firstTokenIds": [ 977, 2262, 62, 4915, 7, 87, 25, 1098 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 2, "commonPrefixTokens": 1, "exactTrajectory": false }, "period_at_s1": { "successDelta": 1, "tokenDelta": -265, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -51, "commonPrefixTokens": 1, "exactTrajectory": false }, "system_at_period": { "successDelta": 1, "tokenDelta": -318, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/133", "index": 2, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 360, "naturalEos": true, "truncated": false, "trajectoryHash": "02a4a5a394bb", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 357, "naturalEos": true, "truncated": false, "trajectoryHash": "1f3227dedff6", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 352, "naturalEos": true, "truncated": false, "trajectoryHash": "657c65932b67", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 78, "naturalEos": true, "truncated": false, "trajectoryHash": "bf4fa0912d02", "firstTokenIds": [ 977, 2555, 62, 72582, 7, 32808, 1780, 185 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -8, "commonPrefixTokens": 25, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -279, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -3, "commonPrefixTokens": 25, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": -274, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/23", "index": 3, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 97, "naturalEos": true, "truncated": false, "trajectoryHash": "fdda57e06407", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 101, "naturalEos": true, "truncated": false, "trajectoryHash": "07b6c2b6a7de", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 327, "naturalEos": true, "truncated": false, "trajectoryHash": "927ab37c7449", "firstTokenIds": [ 429, 1157, 2030, 73675, 63, 317, 4218, 276 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 48, "naturalEos": true, "truncated": false, "trajectoryHash": "f8eebd031ad0", "firstTokenIds": [ 977, 56958, 7, 2612, 25, 1406, 8, 5052 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 230, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -53, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 4, "commonPrefixTokens": 11, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -279, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/147", "index": 4, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 287, "naturalEos": true, "truncated": false, "trajectoryHash": "7968d31e98e1", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 392, "naturalEos": true, "truncated": false, "trajectoryHash": "6d2b0c45c73e", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 442, "naturalEos": true, "truncated": false, "trajectoryHash": "a7ffd6b8c686", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "runtime_error", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "ca6207e139f1", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": 155, "commonPrefixTokens": 16, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 120, "commonPrefixTokens": 32, "exactTrajectory": false }, "system_at_eos": { "successDelta": -1, "tokenDelta": 105, "commonPrefixTokens": 16, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 70, "commonPrefixTokens": 31, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/34", "index": 5, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 207, "naturalEos": true, "truncated": false, "trajectoryHash": "e8306211b557", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 520, 2066 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 173, "naturalEos": true, "truncated": false, "trajectoryHash": "efbee579549f", "firstTokenIds": [ 2158, 2606, 245, 1157, 2030, 19380, 63, 344 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 192, "naturalEos": true, "truncated": false, "trajectoryHash": "2614cbbf28ab", "firstTokenIds": [ 2158, 2606, 245, 1157, 2030, 19380, 63, 344 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 17, "naturalEos": true, "truncated": false, "trajectoryHash": "c52008cb9c22", "firstTokenIds": [ 977, 4730, 7, 75, 25, 1525, 1780, 185 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -15, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -156, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -34, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -175, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/123", "index": 6, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 331, "naturalEos": true, "truncated": false, "trajectoryHash": "349132c5055e", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 543, 4353 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 329, "naturalEos": true, "truncated": false, "trajectoryHash": "92b0903aff50", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 543, 4353 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 507, "naturalEos": true, "truncated": false, "trajectoryHash": "e03d0688f20a", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 543, 4353 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 205, "naturalEos": true, "truncated": false, "trajectoryHash": "92cce4ae0633", "firstTokenIds": [ 977, 752, 62, 16751, 62, 1316, 10243, 89 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 176, "commonPrefixTokens": 25, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -124, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -2, "commonPrefixTokens": 33, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": -302, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/77", "index": 7, "conditions": { "s0_eos": { "success": false, "outcome": "extract_failed", "tokens": 278, "naturalEos": true, "truncated": false, "trajectoryHash": "4a62750de07e", "firstTokenIds": [ 2158, 6983, 565, 274, 10858, 2030, 64, 63 ] }, "s1_eos": { "success": false, "outcome": "runtime_error", "tokens": 303, "naturalEos": true, "truncated": false, "trajectoryHash": "09fa0110978d", "firstTokenIds": [ 2158, 6983, 565, 245, 2028, 10858, 2030, 64 ] }, "s0_period": { "success": false, "outcome": "runtime_error", "tokens": 439, "naturalEos": true, "truncated": false, "trajectoryHash": "cf801d7b12e4", "firstTokenIds": [ 2158, 6983, 565, 245, 2028, 10858, 2030, 64 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 112, "naturalEos": true, "truncated": false, "trajectoryHash": "488b24d54f89", "firstTokenIds": [ 977, 317, 48808, 7, 64, 1780, 185, 300 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 161, "commonPrefixTokens": 3, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -191, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 25, "commonPrefixTokens": 3, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -327, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/22", "index": 8, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 163, "naturalEos": true, "truncated": false, "trajectoryHash": "30cf3dc459e6", "firstTokenIds": [ 4462, 317, 254, 9674, 280, 254, 2030, 8664 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 189, "naturalEos": true, "truncated": false, "trajectoryHash": "b0d6a6b53629", "firstTokenIds": [ 4462, 6, 82, 245, 12974, 1157, 344, 16674 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 212, "naturalEos": true, "truncated": false, "trajectoryHash": "6faf844ccf7c", "firstTokenIds": [ 2158, 4353, 254, 2030, 8664, 62, 6607, 8563 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 211, "naturalEos": true, "truncated": false, "trajectoryHash": "ed3527877810", "firstTokenIds": [ 2158, 6226, 245, 1525, 280, 688, 12974, 3039 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 49, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 22, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 26, "commonPrefixTokens": 1, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -1, "commonPrefixTokens": 1, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "HumanEval/127", "index": 9, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 475, "naturalEos": true, "truncated": false, "trajectoryHash": "ba2bceaaa413", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 468, "naturalEos": true, "truncated": false, "trajectoryHash": "2bd9409092aa", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 439, "naturalEos": true, "truncated": false, "trajectoryHash": "61faa4d92f7e", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 426, "naturalEos": true, "truncated": false, "trajectoryHash": "9e89f0b5ea23", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -36, "commonPrefixTokens": 16, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -42, "commonPrefixTokens": 16, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -7, "commonPrefixTokens": 22, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -13, "commonPrefixTokens": 58, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/37", "index": 10, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 336, "naturalEos": true, "truncated": false, "trajectoryHash": "7f36b1f2ee49", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 2606 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 345, "naturalEos": true, "truncated": false, "trajectoryHash": "94a692a59865", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 2606 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 332, "naturalEos": true, "truncated": false, "trajectoryHash": "98e50a22c918", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 2606 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 117, "naturalEos": true, "truncated": false, "trajectoryHash": "1dde7533a323", "firstTokenIds": [ 977, 3734, 62, 8589, 7, 75, 25, 1525 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -4, "commonPrefixTokens": 14, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -228, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 9, "commonPrefixTokens": 14, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -215, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/48", "index": 11, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 165, "naturalEos": true, "truncated": false, "trajectoryHash": "1f279a162ceb", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 163, "naturalEos": true, "truncated": false, "trajectoryHash": "655d2ec6e291", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 310, "naturalEos": true, "truncated": false, "trajectoryHash": "25de077ab6e7", "firstTokenIds": [ 4462, 6, 82, 254, 12974, 1157, 2030, 262 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 25, "naturalEos": true, "truncated": false, "trajectoryHash": "cf639cf871c2", "firstTokenIds": [ 977, 317, 62, 16435, 515, 7288, 7, 822 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 145, "commonPrefixTokens": 4, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -138, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -2, "commonPrefixTokens": 105, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -285, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/158", "index": 12, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 383, "naturalEos": true, "truncated": false, "trajectoryHash": "ec2ca315c455", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 387, "naturalEos": true, "truncated": false, "trajectoryHash": "64d60661fc00", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 343, "naturalEos": true, "truncated": false, "trajectoryHash": "b3fe45cbd93a", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 385, "naturalEos": true, "truncated": false, "trajectoryHash": "5b5cdc874133", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": -40, "commonPrefixTokens": 29, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -2, "commonPrefixTokens": 8, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 4, "commonPrefixTokens": 41, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 42, "commonPrefixTokens": 8, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/70", "index": 13, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 429, "naturalEos": true, "truncated": false, "trajectoryHash": "6775c6ea0d43", "firstTokenIds": [ 2158, 4353, 254, 2030, 43744, 62, 14318, 62 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 342, "naturalEos": true, "truncated": false, "trajectoryHash": "804179fa4bc8", "firstTokenIds": [ 2158, 4353, 254, 2030, 43744, 62, 14318, 62 ] }, "s0_period": { "success": false, "outcome": "syntax_error", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "1b599cc3a8aa", "firstTokenIds": [ 2158, 4353, 254, 2030, 43744, 62, 14318, 62 ] }, "s1_period": { "success": false, "outcome": "runtime_error", "tokens": 440, "naturalEos": true, "truncated": false, "trajectoryHash": "05f2d4ff1eca", "firstTokenIds": [ 2158, 4353, 254, 2030, 43744, 62, 14318, 62 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 83, "commonPrefixTokens": 12, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": 98, "commonPrefixTokens": 22, "exactTrajectory": false }, "system_at_eos": { "successDelta": 1, "tokenDelta": -87, "commonPrefixTokens": 13, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -72, "commonPrefixTokens": 12, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/89", "index": 14, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 323, "naturalEos": true, "truncated": false, "trajectoryHash": "20cf3156bfba", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 336, "naturalEos": true, "truncated": false, "trajectoryHash": "b49e265f4d94", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 166, "naturalEos": true, "truncated": false, "trajectoryHash": "25ca4d63b9f3", "firstTokenIds": [ 977, 39123, 7, 82, 1780, 185, 300, 23504 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 144, "naturalEos": true, "truncated": false, "trajectoryHash": "3e1eb21230e3", "firstTokenIds": [ 977, 39123, 7, 82, 1780, 185, 300, 23504 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -157, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -192, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 1, "tokenDelta": 13, "commonPrefixTokens": 21, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -22, "commonPrefixTokens": 31, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/9", "index": 15, "conditions": { "s0_eos": { "success": false, "outcome": "runtime_error", "tokens": 171, "naturalEos": true, "truncated": false, "trajectoryHash": "639217e488ee", "firstTokenIds": [ 473, 23134, 1666, 5896, 11, 68776, 185, 185 ] }, "s1_eos": { "success": false, "outcome": "runtime_error", "tokens": 267, "naturalEos": true, "truncated": false, "trajectoryHash": "91444ab53c19", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 481, 938 ] }, "s0_period": { "success": false, "outcome": "runtime_error", "tokens": 109, "naturalEos": true, "truncated": false, "trajectoryHash": "a5f244910716", "firstTokenIds": [ 473, 23134, 1666, 5896, 11, 68776, 185, 185 ] }, "s1_period": { "success": false, "outcome": "runtime_error", "tokens": 109, "naturalEos": true, "truncated": false, "trajectoryHash": "a5f244910716", "firstTokenIds": [ 473, 23134, 1666, 5896, 11, 68776, 185, 185 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -62, "commonPrefixTokens": 8, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -158, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 96, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 109, "exactTrajectory": true } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/53", "index": 16, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 212, "naturalEos": true, "truncated": false, "trajectoryHash": "bf812ffe1064", "firstTokenIds": [ 429, 1157, 2030, 1770, 7, 87, 25, 1098 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 206, "naturalEos": true, "truncated": false, "trajectoryHash": "008f3ad4a12b", "firstTokenIds": [ 429, 1157, 6525, 340, 4286, 317, 29444, 13 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 164, "naturalEos": true, "truncated": false, "trajectoryHash": "08f88dd5e77e", "firstTokenIds": [ 429, 2985, 340, 4286, 317, 245, 1157, 6525 ] }, "s1_period": { "success": false, "outcome": "extract_failed", "tokens": 85, "naturalEos": true, "truncated": false, "trajectoryHash": "94bb19f2316d", "firstTokenIds": [ 429, 1157, 2030, 1770, 63, 4497, 984, 4823 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -48, "commonPrefixTokens": 1, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -121, "commonPrefixTokens": 2, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -6, "commonPrefixTokens": 2, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": -79, "commonPrefixTokens": 1, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "HumanEval/145", "index": 17, "conditions": { "s0_eos": { "success": false, "outcome": "runtime_error", "tokens": 406, "naturalEos": true, "truncated": false, "trajectoryHash": "4def57bad977", "firstTokenIds": [ 2158, 4353, 254, 2030, 2862, 62, 1956, 62 ] }, "s1_eos": { "success": false, "outcome": "runtime_error", "tokens": 398, "naturalEos": true, "truncated": false, "trajectoryHash": "2906305486ea", "firstTokenIds": [ 2158, 4353, 254, 2030, 2862, 62, 1956, 62 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 346, "naturalEos": true, "truncated": false, "trajectoryHash": "ad307492cbf4", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "runtime_error", "tokens": 270, "naturalEos": true, "truncated": false, "trajectoryHash": "2136be2d0e53", "firstTokenIds": [ 977, 1835, 62, 1956, 62, 12140, 7, 6485 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -60, "commonPrefixTokens": 1, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -128, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -8, "commonPrefixTokens": 51, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -76, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/35", "index": 18, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 165, "naturalEos": true, "truncated": false, "trajectoryHash": "bde367e7e902", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 222, "naturalEos": true, "truncated": false, "trajectoryHash": "d5698b92bf4f", "firstTokenIds": [ 2158, 4353, 254, 2030, 3978, 62, 7633, 63 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 174, "naturalEos": true, "truncated": false, "trajectoryHash": "d2d65b7ad848", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 938 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 17, "naturalEos": true, "truncated": false, "trajectoryHash": "30176e3a09e7", "firstTokenIds": [ 977, 3044, 62, 7633, 7, 75, 25, 1525 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 9, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -205, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 57, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -157, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/103", "index": 19, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 416, "naturalEos": true, "truncated": false, "trajectoryHash": "ab0679e0aa86", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 179, "naturalEos": true, "truncated": false, "trajectoryHash": "134c7f287a3b", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 281, "naturalEos": true, "truncated": false, "trajectoryHash": "b0315b03c0b9", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 47, "naturalEos": true, "truncated": false, "trajectoryHash": "cca280f7a5e1", "firstTokenIds": [ 977, 27840, 62, 36744, 7, 77, 11, 273 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -135, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -132, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -237, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -234, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/25", "index": 20, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 404, "naturalEos": true, "truncated": false, "trajectoryHash": "7dfc3c99b18d", "firstTokenIds": [ 2158, 4353, 254, 2030, 24511, 945, 63, 1157 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 465, "naturalEos": true, "truncated": false, "trajectoryHash": "f51a0458d315", "firstTokenIds": [ 2158, 4353, 254, 2030, 24511, 945, 63, 1157 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 425, "naturalEos": true, "truncated": false, "trajectoryHash": "5dea318510b7", "firstTokenIds": [ 2158, 4353, 254, 2030, 24511, 945, 63, 1157 ] }, "s1_period": { "success": false, "outcome": "runtime_error", "tokens": 208, "naturalEos": true, "truncated": false, "trajectoryHash": "e8c5dbc3ee98", "firstTokenIds": [ 977, 6088, 945, 7, 77, 25, 1098, 8 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": 21, "commonPrefixTokens": 34, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -257, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 61, "commonPrefixTokens": 20, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -217, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/98", "index": 21, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 254, "naturalEos": true, "truncated": false, "trajectoryHash": "09f519a955e2", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 239, "naturalEos": true, "truncated": false, "trajectoryHash": "165669786bde", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 2030 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 260, "naturalEos": true, "truncated": false, "trajectoryHash": "20e180c35f11", "firstTokenIds": [ 4462, 6, 82, 245, 12974, 1157, 344, 16839 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 67, "naturalEos": true, "truncated": false, "trajectoryHash": "87962fdd3a88", "firstTokenIds": [ 977, 1690, 62, 22352, 7, 82, 1780, 185 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 6, "commonPrefixTokens": 3, "exactTrajectory": false }, "period_at_s1": { "successDelta": 1, "tokenDelta": -172, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": -1, "tokenDelta": -15, "commonPrefixTokens": 50, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -193, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/115", "index": 22, "conditions": { "s0_eos": { "success": false, "outcome": "runtime_error", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "19ac18a0f9d1", "firstTokenIds": [ 2158, 8708, 254, 2066, 280, 36525, 4627, 45176 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "98781d758100", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 243, "naturalEos": true, "truncated": false, "trajectoryHash": "13109dcb583b", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 456, "naturalEos": true, "truncated": false, "trajectoryHash": "f0851a034394", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -269, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -56, "commonPrefixTokens": 23, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 2, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 213, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/10", "index": 23, "conditions": { "s0_eos": { "success": false, "outcome": "timeout", "tokens": 206, "naturalEos": true, "truncated": false, "trajectoryHash": "98d9783e6b56", "firstTokenIds": [ 4462, 317, 254, 9674, 280, 254, 2030, 7690 ] }, "s1_eos": { "success": false, "outcome": "timeout", "tokens": 213, "naturalEos": true, "truncated": false, "trajectoryHash": "8cbfee1a8652", "firstTokenIds": [ 4462, 317, 254, 9674, 280, 254, 2030, 7690 ] }, "s0_period": { "success": false, "outcome": "timeout", "tokens": 208, "naturalEos": true, "truncated": false, "trajectoryHash": "90b861364699", "firstTokenIds": [ 4462, 317, 254, 9674, 280, 254, 2030, 7690 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 92, "naturalEos": true, "truncated": false, "trajectoryHash": "f128629f477e", "firstTokenIds": [ 977, 1099, 62, 16435, 515, 7288, 7, 2612 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 2, "commonPrefixTokens": 65, "exactTrajectory": false }, "period_at_s1": { "successDelta": 1, "tokenDelta": -121, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 7, "commonPrefixTokens": 107, "exactTrajectory": false }, "system_at_period": { "successDelta": 1, "tokenDelta": -116, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/141", "index": 24, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 496, "naturalEos": true, "truncated": false, "trajectoryHash": "68de6956bb45", "firstTokenIds": [ 2158, 4353, 254, 2030, 2810, 62, 1531, 62 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 290, "naturalEos": true, "truncated": false, "trajectoryHash": "3511784c6023", "firstTokenIds": [ 4462, 6, 82, 245, 12974, 1157, 344, 14180 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 411, "naturalEos": true, "truncated": false, "trajectoryHash": "d8c4fa6913a3", "firstTokenIds": [ 2158, 4353, 254, 2030, 2810, 62, 1531, 62 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 240, "naturalEos": true, "truncated": false, "trajectoryHash": "7d66767a0af1", "firstTokenIds": [ 977, 1761, 62, 1531, 62, 4897, 7, 2810 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -85, "commonPrefixTokens": 28, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -50, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -206, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -171, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "HumanEval/91", "index": 25, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 449, "naturalEos": true, "truncated": false, "trajectoryHash": "ef4c79e23886", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 938 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 316, "naturalEos": true, "truncated": false, "trajectoryHash": "b5a6c981027a", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 938 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 267, "naturalEos": true, "truncated": false, "trajectoryHash": "f3aa2c04d8a6", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 938 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 269, "naturalEos": true, "truncated": false, "trajectoryHash": "1c4b76f6c982", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 340, 481, 938 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -182, "commonPrefixTokens": 23, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -47, "commonPrefixTokens": 15, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -133, "commonPrefixTokens": 23, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 2, "commonPrefixTokens": 15, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/122", "index": 26, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 163, "naturalEos": true, "truncated": false, "trajectoryHash": "a72564638ef4", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 157, "naturalEos": true, "truncated": false, "trajectoryHash": "e044667fa7e1", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 193, "naturalEos": true, "truncated": false, "trajectoryHash": "48ea0f155d34", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 52, "naturalEos": true, "truncated": false, "trajectoryHash": "2e80616effd3", "firstTokenIds": [ 977, 962, 62, 22208, 7, 3049, 11, 530 ] } }, "contrasts": { "period_at_s0": { "successDelta": 1, "tokenDelta": 30, "commonPrefixTokens": 25, "exactTrajectory": false }, "period_at_s1": { "successDelta": 1, "tokenDelta": -105, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -6, "commonPrefixTokens": 62, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -141, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/75", "index": 27, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "fafe77847176", "firstTokenIds": [ 2158, 6983, 565, 245, 2028, 1604, 317, 254 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "7baf64d1d00b", "firstTokenIds": [ 2158, 6983, 565, 245, 2028, 1604, 317, 254 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 182, "naturalEos": true, "truncated": false, "trajectoryHash": "bbba56878e09", "firstTokenIds": [ 977, 317, 62, 84024, 62, 6620, 7, 64 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 234, "naturalEos": true, "truncated": false, "trajectoryHash": "30dfdd28a88a", "firstTokenIds": [ 977, 317, 62, 84024, 62, 6620, 7, 64 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -330, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -278, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 73, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 52, "commonPrefixTokens": 99, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/134", "index": 28, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 390, "naturalEos": true, "truncated": false, "trajectoryHash": "c81cab156949", "firstTokenIds": [ 4462, 6, 82, 254, 9674, 280, 254, 1157 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 248, "naturalEos": true, "truncated": false, "trajectoryHash": "30185955df38", "firstTokenIds": [ 4462, 6, 82, 254, 12974, 2985, 327, 254 ] }, "s0_period": { "success": false, "outcome": "assertion_failed", "tokens": 347, "naturalEos": true, "truncated": false, "trajectoryHash": "a661269c6c22", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 365, "naturalEos": true, "truncated": false, "trajectoryHash": "e14af9de6150", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -43, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 117, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -142, "commonPrefixTokens": 4, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 18, "commonPrefixTokens": 15, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/142", "index": 29, "conditions": { "s0_eos": { "success": false, "outcome": "assertion_failed", "tokens": 481, "naturalEos": true, "truncated": false, "trajectoryHash": "ad25d29cc918", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": false, "outcome": "assertion_failed", "tokens": 448, "naturalEos": true, "truncated": false, "trajectoryHash": "e9fbc617db8a", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 433, "naturalEos": true, "truncated": false, "trajectoryHash": "ffe15a49500a", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 137, "naturalEos": true, "truncated": false, "trajectoryHash": "f68000dbffc5", "firstTokenIds": [ 977, 2555, 62, 72582, 7, 32808, 1780, 185 ] } }, "contrasts": { "period_at_s0": { "successDelta": 1, "tokenDelta": -48, "commonPrefixTokens": 17, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -311, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -33, "commonPrefixTokens": 45, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": -296, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/149", "index": 30, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 388, "naturalEos": true, "truncated": false, "trajectoryHash": "1099cc41e2b4", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 348, "naturalEos": true, "truncated": false, "trajectoryHash": "52f9a00b95da", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 289, "naturalEos": true, "truncated": false, "trajectoryHash": "214637cf12a9", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 132, "naturalEos": true, "truncated": false, "trajectoryHash": "78a43f2715ed", "firstTokenIds": [ 977, 24350, 62, 2502, 62, 1805, 7, 32808 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -99, "commonPrefixTokens": 18, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -216, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -40, "commonPrefixTokens": 18, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -157, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "HumanEval/102", "index": 31, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 214, "naturalEos": true, "truncated": false, "trajectoryHash": "d447e53cf6b5", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 192, "naturalEos": true, "truncated": false, "trajectoryHash": "ddd4e52d97eb", "firstTokenIds": [ 4462, 6, 82, 254, 12974, 2985, 327, 254 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 237, "naturalEos": true, "truncated": false, "trajectoryHash": "4bfba503871a", "firstTokenIds": [ 4462, 317, 245, 12974, 3418, 327, 254, 2066 ] }, "s1_period": { "success": false, "outcome": "assertion_failed", "tokens": 133, "naturalEos": true, "truncated": false, "trajectoryHash": "e4876c9b1fe1", "firstTokenIds": [ 977, 4352, 62, 6485, 7, 87, 11, 320 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 23, "commonPrefixTokens": 29, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -59, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -22, "commonPrefixTokens": 1, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": -104, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] } ], "math": [ { "id": "gsm8k/test/1069", "index": 0, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 206, "naturalEos": true, "truncated": false, "trajectoryHash": "cbf1a78262d6", "firstTokenIds": [ 2195, 1063, 60522, 20570, 56543, 207, 17, 15 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 170, "naturalEos": true, "truncated": false, "trajectoryHash": "830a30d938d1", "firstTokenIds": [ 2195, 1063, 60522, 20570, 56543, 207, 17, 15 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 206, "naturalEos": true, "truncated": false, "trajectoryHash": "c2faeaca9920", "firstTokenIds": [ 2195, 1063, 60522, 20570, 56543, 207, 17, 15 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 186, "naturalEos": true, "truncated": false, "trajectoryHash": "18792fc3b466", "firstTokenIds": [ 2195, 1063, 60522, 20570, 56543, 207, 17, 15 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 24, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 16, "commonPrefixTokens": 53, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -36, "commonPrefixTokens": 24, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -20, "commonPrefixTokens": 53, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "gsm8k/test/1228", "index": 1, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 233, "naturalEos": true, "truncated": false, "trajectoryHash": "2e48132c5b40", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 1604, 280, 27003 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 236, "naturalEos": true, "truncated": false, "trajectoryHash": "c218feb5a172", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 1604, 280, 27003 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 264, "naturalEos": true, "truncated": false, "trajectoryHash": "378dae8e10af", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 1604, 280, 27003 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 270, "naturalEos": true, "truncated": false, "trajectoryHash": "8ab456c76e1d", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 1604, 280, 27003 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 31, "commonPrefixTokens": 94, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 34, "commonPrefixTokens": 11, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 3, "commonPrefixTokens": 172, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 6, "commonPrefixTokens": 11, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0144", "index": 2, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 193, "naturalEos": true, "truncated": false, "trajectoryHash": "201d42085f08", "firstTokenIds": [ 17654, 643, 207, 18, 6287, 6514, 285, 317 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 206, "naturalEos": true, "truncated": false, "trajectoryHash": "b747a7676701", "firstTokenIds": [ 17654, 643, 207, 18, 6287, 6514, 285, 317 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 207, "naturalEos": true, "truncated": false, "trajectoryHash": "1c98102f4c1e", "firstTokenIds": [ 17654, 643, 207, 18, 6287, 6514, 285, 317 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 229, "naturalEos": true, "truncated": false, "trajectoryHash": "d955bcccd11e", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 11462, 254, 3223 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 14, "commonPrefixTokens": 48, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": 23, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 13, "commonPrefixTokens": 48, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": 22, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1251", "index": 3, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 272, "naturalEos": true, "truncated": false, "trajectoryHash": "42e151391e6d", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 282, "naturalEos": true, "truncated": false, "trajectoryHash": "7a7a9679e49c", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 297, "naturalEos": true, "truncated": false, "trajectoryHash": "04248251fa86", "firstTokenIds": [ 3960, 6, 82, 2973, 1224, 254, 766, 359 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 280, "naturalEos": true, "truncated": false, "trajectoryHash": "cb7ccb745dd5", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 933, 276 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": 25, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -2, "commonPrefixTokens": 8, "exactTrajectory": false }, "system_at_eos": { "successDelta": -1, "tokenDelta": 10, "commonPrefixTokens": 17, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -17, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0676", "index": 4, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 168, "naturalEos": true, "truncated": false, "trajectoryHash": "628cc9980e57", "firstTokenIds": [ 53183, 6, 82, 9423, 317, 207, 17, 14 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 147, "naturalEos": true, "truncated": false, "trajectoryHash": "2e79f872a9ef", "firstTokenIds": [ 53183, 6, 82, 9423, 317, 207, 17, 14 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 147, "naturalEos": true, "truncated": false, "trajectoryHash": "9b41a45d50a1", "firstTokenIds": [ 53183, 6, 82, 9423, 438, 207, 17, 14 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 140, "naturalEos": true, "truncated": false, "trajectoryHash": "dc3ff9e25291", "firstTokenIds": [ 53183, 6, 82, 9423, 438, 207, 17, 14 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -21, "commonPrefixTokens": 4, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -7, "commonPrefixTokens": 4, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -21, "commonPrefixTokens": 32, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -7, "commonPrefixTokens": 92, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0126", "index": 5, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 72, "naturalEos": true, "truncated": false, "trajectoryHash": "6f5a09833074", "firstTokenIds": [ 53183, 8939, 207, 18, 23, 14447, 98540, 13 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 72, "naturalEos": true, "truncated": false, "trajectoryHash": "6f5a09833074", "firstTokenIds": [ 53183, 8939, 207, 18, 23, 14447, 98540, 13 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 72, "naturalEos": true, "truncated": false, "trajectoryHash": "6f5a09833074", "firstTokenIds": [ 53183, 8939, 207, 18, 23, 14447, 98540, 13 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 69, "naturalEos": true, "truncated": false, "trajectoryHash": "4513aa0b4ec7", "firstTokenIds": [ 53183, 8939, 207, 18, 23, 14447, 98540, 13 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 72, "exactTrajectory": true }, "period_at_s1": { "successDelta": 0, "tokenDelta": -3, "commonPrefixTokens": 61, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 72, "exactTrajectory": true }, "system_at_period": { "successDelta": 0, "tokenDelta": -3, "commonPrefixTokens": 61, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0438", "index": 6, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 359, "naturalEos": true, "truncated": false, "trajectoryHash": "4496e6fd6e30", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 305, "naturalEos": true, "truncated": false, "trajectoryHash": "04fca59a3357", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 252, "naturalEos": true, "truncated": false, "trajectoryHash": "d080eda4535c", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 245, "naturalEos": true, "truncated": false, "trajectoryHash": "aa46af5038f0", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -107, "commonPrefixTokens": 15, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -60, "commonPrefixTokens": 15, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -54, "commonPrefixTokens": 32, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -7, "commonPrefixTokens": 104, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1299", "index": 7, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 111, "naturalEos": true, "truncated": false, "trajectoryHash": "2fa8c2f0d99d", "firstTokenIds": [ 40804, 41504, 363, 18, 13, 20, 15, 331 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 107, "naturalEos": true, "truncated": false, "trajectoryHash": "bc76b9b932ad", "firstTokenIds": [ 40804, 41504, 363, 18, 13, 20, 15, 331 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 111, "naturalEos": true, "truncated": false, "trajectoryHash": "1621f5c8b919", "firstTokenIds": [ 40804, 41504, 363, 18, 13, 20, 15, 331 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 106, "naturalEos": true, "truncated": false, "trajectoryHash": "57c5b323b6e0", "firstTokenIds": [ 40804, 41504, 363, 18, 13, 20, 15, 331 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 69, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -1, "commonPrefixTokens": 37, "exactTrajectory": false }, "system_at_eos": { "successDelta": 1, "tokenDelta": -4, "commonPrefixTokens": 57, "exactTrajectory": false }, "system_at_period": { "successDelta": 1, "tokenDelta": -5, "commonPrefixTokens": 37, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0546", "index": 8, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 120, "naturalEos": true, "truncated": false, "trajectoryHash": "dc49a0944bb1", "firstTokenIds": [ 93435, 14870, 254, 32628, 279, 2201, 9682, 11 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 94, "naturalEos": true, "truncated": false, "trajectoryHash": "c2793629ec5b", "firstTokenIds": [ 93435, 14870, 254, 32628, 279, 2201, 9682, 11 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 107, "naturalEos": true, "truncated": false, "trajectoryHash": "b9ce7eb06d11", "firstTokenIds": [ 93435, 14870, 254, 32628, 9682, 11, 588, 317 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 107, "naturalEos": true, "truncated": false, "trajectoryHash": "71deb875dfce", "firstTokenIds": [ 93435, 14870, 254, 3912, 32628, 9682, 11, 588 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": -13, "commonPrefixTokens": 4, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 13, "commonPrefixTokens": 3, "exactTrajectory": false }, "system_at_eos": { "successDelta": -1, "tokenDelta": -26, "commonPrefixTokens": 34, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 3, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "gsm8k/test/0537", "index": 9, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 122, "naturalEos": true, "truncated": false, "trajectoryHash": "95723a6a422a", "firstTokenIds": [ 429, 1022, 2135, 317, 207, 18, 16, 11237 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 122, "naturalEos": true, "truncated": false, "trajectoryHash": "95723a6a422a", "firstTokenIds": [ 429, 1022, 2135, 317, 207, 18, 16, 11237 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 122, "naturalEos": true, "truncated": false, "trajectoryHash": "95723a6a422a", "firstTokenIds": [ 429, 1022, 2135, 317, 207, 18, 16, 11237 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 181, "naturalEos": true, "truncated": false, "trajectoryHash": "bcb6e4ca9af6", "firstTokenIds": [ 2158, 1275, 254, 3223, 22321, 280, 254, 2685 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 122, "exactTrajectory": true }, "period_at_s1": { "successDelta": 0, "tokenDelta": 59, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 122, "exactTrajectory": true }, "system_at_period": { "successDelta": 0, "tokenDelta": 59, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0226", "index": 10, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 191, "naturalEos": true, "truncated": false, "trajectoryHash": "85ab0782cea2", "firstTokenIds": [ 7518, 15920, 643, 207, 18, 15, 4043, 1449 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 328, "naturalEos": true, "truncated": false, "trajectoryHash": "d4fe2f6566c5", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 310, "naturalEos": true, "truncated": false, "trajectoryHash": "8ea8fd539f3d", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 251, "naturalEos": true, "truncated": false, "trajectoryHash": "eeebfcc26f88", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 119, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -77, "commonPrefixTokens": 43, "exactTrajectory": false }, "system_at_eos": { "successDelta": 1, "tokenDelta": 137, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -59, "commonPrefixTokens": 43, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1128", "index": 11, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 124, "naturalEos": true, "truncated": false, "trajectoryHash": "409230e40c0b", "firstTokenIds": [ 3960, 6, 82, 9211, 76282, 6, 82, 1648 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 126, "naturalEos": true, "truncated": false, "trajectoryHash": "5ddd77291ff1", "firstTokenIds": [ 3960, 6, 82, 9211, 76282, 6, 82, 1648 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 128, "naturalEos": true, "truncated": false, "trajectoryHash": "012b66fdfbc3", "firstTokenIds": [ 3960, 6, 82, 9211, 76282, 6, 82, 1648 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 146, "naturalEos": true, "truncated": false, "trajectoryHash": "4b964e74637c", "firstTokenIds": [ 3960, 6, 82, 9211, 76282, 6, 82, 1648 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 4, "commonPrefixTokens": 59, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 20, "commonPrefixTokens": 19, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 2, "commonPrefixTokens": 56, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 18, "commonPrefixTokens": 19, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0760", "index": 12, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 157, "naturalEos": true, "truncated": false, "trajectoryHash": "be807c966edf", "firstTokenIds": [ 207, 17, 15, 4, 280, 207, 20, 15 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 157, "naturalEos": true, "truncated": false, "trajectoryHash": "be807c966edf", "firstTokenIds": [ 207, 17, 15, 4, 280, 207, 20, 15 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 150, "naturalEos": true, "truncated": false, "trajectoryHash": "5406ccd886aa", "firstTokenIds": [ 207, 17, 15, 4, 280, 207, 20, 15 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 179, "naturalEos": true, "truncated": false, "trajectoryHash": "c6a74b86ecae", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -7, "commonPrefixTokens": 112, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 22, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 157, "exactTrajectory": true }, "system_at_period": { "successDelta": 0, "tokenDelta": 29, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1173", "index": 13, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 163, "naturalEos": true, "truncated": false, "trajectoryHash": "58d6f7c3c141", "firstTokenIds": [ 429, 4252, 4085, 280, 54078, 57139, 7025, 363 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 248, "naturalEos": true, "truncated": false, "trajectoryHash": "3465cd120aa6", "firstTokenIds": [ 429, 4252, 4085, 280, 54078, 57139, 7025, 363 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 221, "naturalEos": true, "truncated": false, "trajectoryHash": "089b76070587", "firstTokenIds": [ 429, 4252, 4085, 280, 54078, 57139, 7025, 363 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 199, "naturalEos": true, "truncated": false, "trajectoryHash": "f6e0428ea361", "firstTokenIds": [ 429, 4252, 4085, 280, 54078, 57139, 7025, 363 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": 58, "commonPrefixTokens": 17, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -49, "commonPrefixTokens": 10, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 85, "commonPrefixTokens": 41, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -22, "commonPrefixTokens": 10, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0452", "index": 14, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 254, "naturalEos": true, "truncated": false, "trajectoryHash": "a0ca2283faac", "firstTokenIds": [ 2158, 1275, 254, 5137, 4583, 11, 395, 933 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 254, "naturalEos": true, "truncated": false, "trajectoryHash": "be5b1aa682cd", "firstTokenIds": [ 2158, 1275, 254, 5137, 4583, 11, 395, 933 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 254, "naturalEos": true, "truncated": false, "trajectoryHash": "a0ca2283faac", "firstTokenIds": [ 2158, 1275, 254, 5137, 4583, 11, 395, 933 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 246, "naturalEos": true, "truncated": false, "trajectoryHash": "41b1e30b7303", "firstTokenIds": [ 2158, 1275, 254, 5137, 4583, 11, 395, 933 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 254, "exactTrajectory": true }, "period_at_s1": { "successDelta": 0, "tokenDelta": -8, "commonPrefixTokens": 152, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 213, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -8, "commonPrefixTokens": 152, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0357", "index": 15, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 299, "naturalEos": true, "truncated": false, "trajectoryHash": "8af193f7a509", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 11462, 254, 3223 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 399, "naturalEos": true, "truncated": false, "trajectoryHash": "d38f3ac8571b", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 11462, 254, 3223 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 309, "naturalEos": true, "truncated": false, "trajectoryHash": "a704c7519b3d", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 11462, 254, 3223 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 323, "naturalEos": true, "truncated": false, "trajectoryHash": "62a9d6f1afb1", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 11462, 254, 3223 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 10, "commonPrefixTokens": 44, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -76, "commonPrefixTokens": 14, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 100, "commonPrefixTokens": 50, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 14, "commonPrefixTokens": 14, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1050", "index": 16, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 88, "naturalEos": true, "truncated": false, "trajectoryHash": "fbf4500cf75d", "firstTokenIds": [ 11967, 578, 17354, 207, 18, 17, 20, 49785 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 122, "naturalEos": true, "truncated": false, "trajectoryHash": "ecbd88f6d536", "firstTokenIds": [ 11967, 578, 17354, 207, 18, 17, 20, 49785 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 88, "naturalEos": true, "truncated": false, "trajectoryHash": "fbf4500cf75d", "firstTokenIds": [ 11967, 578, 17354, 207, 18, 17, 20, 49785 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 125, "naturalEos": true, "truncated": false, "trajectoryHash": "004624477d47", "firstTokenIds": [ 11967, 578, 643, 2571, 11516, 207, 18, 17 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 88, "exactTrajectory": true }, "period_at_s1": { "successDelta": -1, "tokenDelta": 3, "commonPrefixTokens": 2, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 34, "commonPrefixTokens": 52, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": 37, "commonPrefixTokens": 2, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "gsm8k/test/0951", "index": 17, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 54, "naturalEos": true, "truncated": false, "trajectoryHash": "15362625ba4b", "firstTokenIds": [ 1273, 745, 418, 207, 16, 15, 15, 6769 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 67, "naturalEos": true, "truncated": false, "trajectoryHash": "8fc35b0abdd6", "firstTokenIds": [ 1273, 745, 418, 207, 16, 15, 15, 6769 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 107, "naturalEos": true, "truncated": false, "trajectoryHash": "256c27659717", "firstTokenIds": [ 1273, 745, 418, 207, 16, 15, 15, 6769 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 232, "naturalEos": true, "truncated": false, "trajectoryHash": "8c88a0c51e11", "firstTokenIds": [ 2158, 1275, 254, 1604, 280, 6769, 279, 254 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 53, "commonPrefixTokens": 25, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 165, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 13, "commonPrefixTokens": 41, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 125, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0906", "index": 18, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 136, "naturalEos": true, "truncated": false, "trajectoryHash": "345cd606d60c", "firstTokenIds": [ 9663, 643, 207, 22, 5501, 24948, 14253, 285 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 191, "naturalEos": true, "truncated": false, "trajectoryHash": "d9d7eb803d59", "firstTokenIds": [ 9663, 643, 207, 22, 5501, 24948, 14253, 285 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 127, "naturalEos": true, "truncated": false, "trajectoryHash": "3277de277146", "firstTokenIds": [ 9663, 643, 245, 3223, 280, 207, 22, 919 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 223, "naturalEos": true, "truncated": false, "trajectoryHash": "7755880ab652", "firstTokenIds": [ 2158, 8708, 437, 2066, 11, 395, 1022, 933 ] } }, "contrasts": { "period_at_s0": { "successDelta": 1, "tokenDelta": -9, "commonPrefixTokens": 2, "exactTrajectory": false }, "period_at_s1": { "successDelta": 1, "tokenDelta": 32, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 55, "commonPrefixTokens": 41, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 96, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0092", "index": 19, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 153, "naturalEos": true, "truncated": false, "trajectoryHash": "381e09cb9a85", "firstTokenIds": [ 1273, 7518, 317, 207, 16, 15, 285, 317 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 98, "naturalEos": true, "truncated": false, "trajectoryHash": "8dea3bbc7ac3", "firstTokenIds": [ 1273, 7518, 317, 207, 16, 15, 285, 317 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 82, "naturalEos": true, "truncated": false, "trajectoryHash": "5fa78969ef22", "firstTokenIds": [ 1273, 7518, 317, 207, 16, 15, 285, 317 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 131, "naturalEos": true, "truncated": false, "trajectoryHash": "d1f7d1f3de0d", "firstTokenIds": [ 3960, 6, 82, 8708, 437, 3458, 457, 3458 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": -71, "commonPrefixTokens": 42, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 33, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": -1, "tokenDelta": -55, "commonPrefixTokens": 63, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 49, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0372", "index": 20, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 136, "naturalEos": true, "truncated": false, "trajectoryHash": "ae3f631d9905", "firstTokenIds": [ 9243, 522, 661, 12116, 5561, 327, 207, 16 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 136, "naturalEos": true, "truncated": false, "trajectoryHash": "b741538c8f6a", "firstTokenIds": [ 9243, 522, 661, 12116, 5561, 327, 207, 16 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 136, "naturalEos": true, "truncated": false, "trajectoryHash": "b741538c8f6a", "firstTokenIds": [ 9243, 522, 661, 12116, 5561, 327, 207, 16 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 144, "naturalEos": true, "truncated": false, "trajectoryHash": "206c3c33c1a7", "firstTokenIds": [ 9243, 522, 661, 12116, 5561, 327, 207, 16 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 63, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 8, "commonPrefixTokens": 47, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 63, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 8, "commonPrefixTokens": 47, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0361", "index": 21, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 256, "naturalEos": true, "truncated": false, "trajectoryHash": "cee4bc203483", "firstTokenIds": [ 429, 6822, 29099, 207, 22, 20, 6593, 473 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 257, "naturalEos": true, "truncated": false, "trajectoryHash": "adcdcd6996b9", "firstTokenIds": [ 429, 6822, 29099, 207, 22, 20, 6593, 473 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 196, "naturalEos": true, "truncated": false, "trajectoryHash": "1939e08c4ded", "firstTokenIds": [ 429, 6822, 29099, 207, 22, 20, 6593, 473 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 242, "naturalEos": true, "truncated": false, "trajectoryHash": "190248c44054", "firstTokenIds": [ 429, 6822, 29099, 207, 22, 20, 6593, 473 ] } }, "contrasts": { "period_at_s0": { "successDelta": 1, "tokenDelta": -60, "commonPrefixTokens": 52, "exactTrajectory": false }, "period_at_s1": { "successDelta": 1, "tokenDelta": -15, "commonPrefixTokens": 15, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 1, "commonPrefixTokens": 15, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 46, "commonPrefixTokens": 52, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0733", "index": 22, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 83, "naturalEos": true, "truncated": false, "trajectoryHash": "f65911fec2a3", "firstTokenIds": [ 17157, 1503, 363, 17, 15, 285, 362, 7572 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 83, "naturalEos": true, "truncated": false, "trajectoryHash": "f65911fec2a3", "firstTokenIds": [ 17157, 1503, 363, 17, 15, 285, 362, 7572 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 83, "naturalEos": true, "truncated": false, "trajectoryHash": "f65911fec2a3", "firstTokenIds": [ 17157, 1503, 363, 17, 15, 285, 362, 7572 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 81, "naturalEos": true, "truncated": false, "trajectoryHash": "41adcba6b4f3", "firstTokenIds": [ 2158, 1275, 636, 946, 1266, 3054, 1319, 60282 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 83, "exactTrajectory": true }, "period_at_s1": { "successDelta": 0, "tokenDelta": -2, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 83, "exactTrajectory": true }, "system_at_period": { "successDelta": 0, "tokenDelta": -2, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0687", "index": 23, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 282, "naturalEos": true, "truncated": false, "trajectoryHash": "0c81b2f9ed65", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 4500, 280, 634 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 217, "naturalEos": true, "truncated": false, "trajectoryHash": "a179ca00e6cd", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 4500, 280, 634 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 234, "naturalEos": true, "truncated": false, "trajectoryHash": "30bdc63f93ea", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 4500, 280, 634 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 202, "naturalEos": true, "truncated": false, "trajectoryHash": "3f9b54299aee", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 13205, 280, 254 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -48, "commonPrefixTokens": 8, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -15, "commonPrefixTokens": 5, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -65, "commonPrefixTokens": 8, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": -32, "commonPrefixTokens": 5, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0584", "index": 24, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 298, "naturalEos": true, "truncated": false, "trajectoryHash": "9dce4686d271", "firstTokenIds": [ 3960, 6, 82, 2973, 1224, 254, 2066, 3458 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 314, "naturalEos": true, "truncated": false, "trajectoryHash": "93961697cdf4", "firstTokenIds": [ 3960, 6, 82, 2973, 1224, 254, 2066, 3458 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 290, "naturalEos": true, "truncated": false, "trajectoryHash": "d8bf77d7440c", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 284, "naturalEos": true, "truncated": false, "trajectoryHash": "da9fed9acdd7", "firstTokenIds": [ 5904, 11, 395, 933, 276, 6983, 946, 1313 ] } }, "contrasts": { "period_at_s0": { "successDelta": 1, "tokenDelta": -8, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": -1, "tokenDelta": -30, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 1, "tokenDelta": 16, "commonPrefixTokens": 10, "exactTrajectory": false }, "system_at_period": { "successDelta": -1, "tokenDelta": -6, "commonPrefixTokens": 2, "exactTrajectory": false } }, "diagnosticTapes": [ "T0", "T1", "T2", "T3" ] }, { "id": "gsm8k/test/1152", "index": 25, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 149, "naturalEos": true, "truncated": false, "trajectoryHash": "afc53c199efe", "firstTokenIds": [ 429, 1022, 48441, 15006, 643, 245, 207, 17 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 244, "naturalEos": true, "truncated": false, "trajectoryHash": "cc57dc1d59c9", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 7599, 280, 254 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 176, "naturalEos": true, "truncated": false, "trajectoryHash": "f80e45684bc5", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 7599, 280, 254 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 196, "naturalEos": true, "truncated": false, "trajectoryHash": "2600b7dd1901", "firstTokenIds": [ 3960, 6, 82, 2973, 1224, 254, 2066, 25 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 27, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -48, "commonPrefixTokens": 3, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 95, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 20, "commonPrefixTokens": 3, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1021", "index": 26, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 190, "naturalEos": true, "truncated": false, "trajectoryHash": "6447b9536055", "firstTokenIds": [ 207, 19, 15, 4, 280, 254, 4313, 418 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 180, "naturalEos": true, "truncated": false, "trajectoryHash": "98267904a531", "firstTokenIds": [ 207, 19, 15, 4, 280, 254, 4313, 418 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 152, "naturalEos": true, "truncated": false, "trajectoryHash": "ead34de5c2cf", "firstTokenIds": [ 338, 2228, 317, 2049, 276, 752, 207, 22 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 297, "naturalEos": true, "truncated": false, "trajectoryHash": "a3792c95d26a", "firstTokenIds": [ 5904, 11, 1632, 6, 82, 1275, 636, 946 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -38, "commonPrefixTokens": 0, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 117, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -10, "commonPrefixTokens": 71, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 145, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1194", "index": 27, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 97, "naturalEos": true, "truncated": false, "trajectoryHash": "c9bbda61f523", "firstTokenIds": [ 1273, 245, 51885, 643, 1853, 2602, 372, 1266 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 97, "naturalEos": true, "truncated": false, "trajectoryHash": "c9bbda61f523", "firstTokenIds": [ 1273, 245, 51885, 643, 1853, 2602, 372, 1266 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 78, "naturalEos": true, "truncated": false, "trajectoryHash": "838cb016b96a", "firstTokenIds": [ 1273, 245, 51885, 643, 1853, 2602, 372, 1266 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 209, "naturalEos": true, "truncated": false, "trajectoryHash": "2d8a1d1b7c5a", "firstTokenIds": [ 3960, 6, 82, 2973, 1224, 254, 2066, 881 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -19, "commonPrefixTokens": 68, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 112, "commonPrefixTokens": 0, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 97, "exactTrajectory": true }, "system_at_period": { "successDelta": 0, "tokenDelta": 131, "commonPrefixTokens": 0, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0399", "index": 28, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 85, "naturalEos": true, "truncated": false, "trajectoryHash": "736f1945483a", "firstTokenIds": [ 1273, 82272, 317, 207, 18, 15, 1555, 1712 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 86, "naturalEos": true, "truncated": false, "trajectoryHash": "fdd7523b771b", "firstTokenIds": [ 1273, 82272, 317, 207, 18, 15, 1555, 1712 ] }, "s0_period": { "success": true, "outcome": "passed", "tokens": 86, "naturalEos": true, "truncated": false, "trajectoryHash": "fdd7523b771b", "firstTokenIds": [ 1273, 82272, 317, 207, 18, 15, 1555, 1712 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 86, "naturalEos": true, "truncated": false, "trajectoryHash": "fdd7523b771b", "firstTokenIds": [ 1273, 82272, 317, 207, 18, 15, 1555, 1712 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 1, "commonPrefixTokens": 28, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 86, "exactTrajectory": true }, "system_at_eos": { "successDelta": 0, "tokenDelta": 1, "commonPrefixTokens": 28, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 86, "exactTrajectory": true } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0214", "index": 29, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 512, "naturalEos": false, "truncated": true, "trajectoryHash": "6820a91df8d1", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 3744, 280, 2354 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 494, "naturalEos": true, "truncated": false, "trajectoryHash": "c61d867bb525", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 3744, 280, 2354 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 308, "naturalEos": true, "truncated": false, "trajectoryHash": "318257aed6de", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 3744, 280, 2354 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 471, "naturalEos": true, "truncated": false, "trajectoryHash": "60e00d082884", "firstTokenIds": [ 3960, 6, 82, 9211, 254, 3744, 280, 2354 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": -204, "commonPrefixTokens": 94, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -23, "commonPrefixTokens": 65, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": -18, "commonPrefixTokens": 12, "exactTrajectory": false }, "system_at_period": { "successDelta": 0, "tokenDelta": 163, "commonPrefixTokens": 12, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/1084", "index": 30, "conditions": { "s0_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 174, "naturalEos": true, "truncated": false, "trajectoryHash": "e19fa9207aa3", "firstTokenIds": [ 3960, 6, 82, 6049, 7421, 5698, 1376, 5144 ] }, "s1_eos": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 174, "naturalEos": true, "truncated": false, "trajectoryHash": "e19fa9207aa3", "firstTokenIds": [ 3960, 6, 82, 6049, 7421, 5698, 1376, 5144 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 174, "naturalEos": true, "truncated": false, "trajectoryHash": "e19fa9207aa3", "firstTokenIds": [ 3960, 6, 82, 6049, 7421, 5698, 1376, 5144 ] }, "s1_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 188, "naturalEos": true, "truncated": false, "trajectoryHash": "540b2e41129b", "firstTokenIds": [ 3960, 6, 82, 6049, 7421, 5698, 1376, 5144 ] } }, "contrasts": { "period_at_s0": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 174, "exactTrajectory": true }, "period_at_s1": { "successDelta": 0, "tokenDelta": 14, "commonPrefixTokens": 71, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 0, "commonPrefixTokens": 174, "exactTrajectory": true }, "system_at_period": { "successDelta": 0, "tokenDelta": 14, "commonPrefixTokens": 71, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] }, { "id": "gsm8k/test/0964", "index": 31, "conditions": { "s0_eos": { "success": true, "outcome": "passed", "tokens": 68, "naturalEos": true, "truncated": false, "trajectoryHash": "323a943a78a8", "firstTokenIds": [ 1273, 1319, 15341, 317, 207, 17, 15, 4063 ] }, "s1_eos": { "success": true, "outcome": "passed", "tokens": 98, "naturalEos": true, "truncated": false, "trajectoryHash": "5b09328f0ddd", "firstTokenIds": [ 1273, 1319, 15341, 317, 207, 17, 15, 4063 ] }, "s0_period": { "success": false, "outcome": "wrong_numeric_answer", "tokens": 68, "naturalEos": true, "truncated": false, "trajectoryHash": "24e0750594b9", "firstTokenIds": [ 1273, 1319, 15341, 317, 207, 17, 15, 4063 ] }, "s1_period": { "success": true, "outcome": "passed", "tokens": 78, "naturalEos": true, "truncated": false, "trajectoryHash": "6ad7b8c26f21", "firstTokenIds": [ 1273, 1319, 15341, 317, 207, 17, 15, 4063 ] } }, "contrasts": { "period_at_s0": { "successDelta": -1, "tokenDelta": 0, "commonPrefixTokens": 16, "exactTrajectory": false }, "period_at_s1": { "successDelta": 0, "tokenDelta": -20, "commonPrefixTokens": 9, "exactTrajectory": false }, "system_at_eos": { "successDelta": 0, "tokenDelta": 30, "commonPrefixTokens": 17, "exactTrajectory": false }, "system_at_period": { "successDelta": 1, "tokenDelta": 10, "commonPrefixTokens": 9, "exactTrajectory": false } }, "diagnosticTapes": [ "T0" ] } ] }, "outcomes": { "code": { "assertion_failed": 47, "extract_failed": 2, "passed": 59, "runtime_error": 16, "syntax_error": 1, "timeout": 3 }, "math": { "passed": 71, "wrong_numeric_answer": 57 } }, "reproduction": { "cells": 64, "all_preregistered_fields_exact": 64, "by_field": { "run_seed_exact": 64, "prompt_hash_exact": 64, "uniform_run_hash_exact": 64, "uniform_output_prefix_hash_exact": 64, "uniform_steps_exact": 64, "generated_token_ids_exact": 64, "decoded_text_exact": 64, "eos_state_exact": 64, "truncation_state_exact": 64, "cpu_rng_pre_state_exact": 64, "cuda_rng_pre_state_exact": 64, "torch_rng_unchanged_exact": 64 } }, "bootstrap": { "resamples": 10000, "seed": 1364512825, "rng": "numpy.random.default_rng reset per domain", "unit": "selected task", "paired_conditions": true, "tape": "T0", "interval_label": "selected-task resampling band for the fixed 32-task frame and T0", "not": [ "benchmark-population confidence interval", "model-ability confidence interval", "generation-seed uncertainty interval", "causal-effect confidence interval" ] }, "uniformExample": { "sourceId": "HumanEval/31", "tape": "T0", "uniformUint64FirstEightHex": [ "72223be6d80669ef", "41a829ee24b9f837", "1fc9148ea2f2ac2c", "f6349bcb6cec7b5b", "bd0b252f46fec99d", "ae62463483fd351b", "836e54d1468e348e", "e47b1da3da2685d8" ], "uniformFloat32FirstEight": [ 0.4458348751068115, 0.25647222995758057, 0.1241619884967804, 0.961740255355835, 0.7384513020515442, 0.6811870336532593, 0.5134022831916809, 0.8925036191940308 ], "conditions": { "s0_eos": { "generatedTokenIds": [ 2158, 4353, 254, 2030, 262, 62, 6620, 63 ], "generatedTokens": 394, "naturalEos": true }, "s1_eos": { "generatedTokenIds": [ 2158, 6983, 565, 245, 1604, 317, 9966, 11 ], "generatedTokens": 344, "naturalEos": true }, "s0_period": { "generatedTokenIds": [ 2158, 4353, 254, 2030, 262, 62, 6620, 63 ], "generatedTokens": 420, "naturalEos": true }, "s1_period": { "generatedTokenIds": [ 977, 317, 62, 6620, 7, 77, 1780, 185 ], "generatedTokens": 59, "naturalEos": true } } }, "deviations": [ { "id": "prompt-hash-correction", "severity": "corrected-before-output", "summary": "The first manifest mislabeled routing-probe hashes as Chat prompt hashes; all 256 Chat hashes were corrected before model output." }, { "id": "gold-loaded-in-runner", "severity": "reported-process-deviation", "summary": "The reused runner loaded gold for a post-decode narrow task_score before generation finished. Gold never entered prompts, logits, sampling, selection, or the authoritative evaluator." } ], "evidenceBoundary": [ "HumanEval and GSM8K remain separate.", "Selected-task bands cover only the frozen 32-task frame under T0.", "T1-T3 are sensitivity diagnostics, not extra independent tasks.", "The explicit sampler uses the official .3/.95 distribution but is not a torch.multinomial trajectory.", "Period prompts are counterfactual and not official-valid chats." ] }