wuyang
8699 lines
209 KiB
JSON
8699 lines
209 KiB
JSON
{
|
|
"schemaVersion": 1,
|
|
"protocolId": "llm-atlas-deepseek-chat-task-bootstrap-crn-v1",
|
|
"capturedAt": "2026-07-29T20:25:26.335344+00:00",
|
|
"model": {
|
|
"repo": "deepseek-ai/DeepSeek-V2-Lite-Chat",
|
|
"revision": "85864749cd611b4353ce1decdb286193298f64c7",
|
|
"checkpointIdentity": "SFT Chat",
|
|
"dtype": "bfloat16"
|
|
},
|
|
"artifactHashes": {
|
|
"sampling": "ea0607f2b197fac3f794655c1538ce1f9a1cb072d637eb31d35573682e311809",
|
|
"evaluation": "82b2fc5d1f854a7e0cd7aba7c70ff74d733d24221522a4f92b962478c76177ab",
|
|
"reproduction": "63ed39e5dcdbc2a30e516172f3657dfa453ff3b23d5bd5c49c734939242a70f5",
|
|
"analysis": "9ab17561ced930a668c082141f7c6e013cbda70e42b09de63d41f1b82c01a6ae"
|
|
},
|
|
"grid": {
|
|
"formalSources": 64,
|
|
"formalRuns": 88,
|
|
"formalOutputs": 352,
|
|
"mainT0Outputs": 256,
|
|
"diagnosticAdditionalOutputs": 96,
|
|
"naturalEos": 343,
|
|
"budgetTruncated": 9,
|
|
"uniqueTrajectories": 320,
|
|
"promptHashesExact": 256,
|
|
"torchRngUnchangedRuns": 88
|
|
},
|
|
"sampler": {
|
|
"name": "explicit-uniform inverse-CDF nucleus sampler in token-ID order",
|
|
"temperature": 0.3,
|
|
"topP": 0.95,
|
|
"softmaxDtype": "torch.float32",
|
|
"cdfDtype": "torch.float32",
|
|
"uniformDtype": "torch.float32",
|
|
"transformersGenerateCalled": false,
|
|
"torchMultinomialCalled": false,
|
|
"commonRandomNumbers": true
|
|
},
|
|
"conditionTable": {
|
|
"code": [
|
|
{
|
|
"condition": "s0_eos",
|
|
"outputs": 32,
|
|
"success": 16,
|
|
"successRate": 0.5,
|
|
"naturalEos": 30,
|
|
"naturalEosRate": 0.9375,
|
|
"meanTokens": 330.09375,
|
|
"outcomes": {
|
|
"assertion_failed": 10,
|
|
"extract_failed": 1,
|
|
"passed": 16,
|
|
"runtime_error": 4,
|
|
"timeout": 1
|
|
}
|
|
},
|
|
{
|
|
"condition": "s1_eos",
|
|
"outputs": 32,
|
|
"success": 16,
|
|
"successRate": 0.5,
|
|
"naturalEos": 30,
|
|
"naturalEosRate": 0.9375,
|
|
"meanTokens": 308.9375,
|
|
"outcomes": {
|
|
"assertion_failed": 11,
|
|
"passed": 16,
|
|
"runtime_error": 4,
|
|
"timeout": 1
|
|
}
|
|
},
|
|
{
|
|
"condition": "s0_period",
|
|
"outputs": 32,
|
|
"success": 15,
|
|
"successRate": 0.46875,
|
|
"naturalEos": 31,
|
|
"naturalEosRate": 0.96875,
|
|
"meanTokens": 314.375,
|
|
"outcomes": {
|
|
"assertion_failed": 12,
|
|
"passed": 15,
|
|
"runtime_error": 3,
|
|
"syntax_error": 1,
|
|
"timeout": 1
|
|
}
|
|
},
|
|
{
|
|
"condition": "s1_period",
|
|
"outputs": 32,
|
|
"success": 12,
|
|
"successRate": 0.375,
|
|
"naturalEos": 31,
|
|
"naturalEosRate": 0.96875,
|
|
"meanTokens": 183.5,
|
|
"outcomes": {
|
|
"assertion_failed": 14,
|
|
"extract_failed": 1,
|
|
"passed": 12,
|
|
"runtime_error": 5
|
|
}
|
|
}
|
|
],
|
|
"math": [
|
|
{
|
|
"condition": "s0_eos",
|
|
"outputs": 32,
|
|
"success": 19,
|
|
"successRate": 0.59375,
|
|
"naturalEos": 31,
|
|
"naturalEosRate": 0.96875,
|
|
"meanTokens": 181.40625,
|
|
"outcomes": {
|
|
"passed": 19,
|
|
"wrong_numeric_answer": 13
|
|
}
|
|
},
|
|
{
|
|
"condition": "s1_eos",
|
|
"outputs": 32,
|
|
"success": 19,
|
|
"successRate": 0.59375,
|
|
"naturalEos": 32,
|
|
"naturalEosRate": 1,
|
|
"meanTokens": 190.96875,
|
|
"outcomes": {
|
|
"passed": 19,
|
|
"wrong_numeric_answer": 13
|
|
}
|
|
},
|
|
{
|
|
"condition": "s0_period",
|
|
"outputs": 32,
|
|
"success": 17,
|
|
"successRate": 0.53125,
|
|
"naturalEos": 32,
|
|
"naturalEosRate": 1,
|
|
"meanTokens": 173.1875,
|
|
"outcomes": {
|
|
"passed": 17,
|
|
"wrong_numeric_answer": 15
|
|
}
|
|
},
|
|
{
|
|
"condition": "s1_period",
|
|
"outputs": 32,
|
|
"success": 16,
|
|
"successRate": 0.5,
|
|
"naturalEos": 32,
|
|
"naturalEosRate": 1,
|
|
"meanTokens": 198.3125,
|
|
"outcomes": {
|
|
"passed": 16,
|
|
"wrong_numeric_answer": 16
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"contrasts": {
|
|
"code": {
|
|
"period_at_s0": {
|
|
"left": "s0_eos",
|
|
"right": "s0_period",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": -0.03125,
|
|
"band": {
|
|
"p2_5": -0.15625,
|
|
"p50": -0.03125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 27,
|
|
"negative": 3
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 14,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 3,
|
|
"pass_to_pass": 13
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": -0.03125,
|
|
"band": {
|
|
"p2_5": -0.15625,
|
|
"p50": -0.03125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 27,
|
|
"negative": 3
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 14,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 3,
|
|
"pass_to_pass": 13
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0.03125,
|
|
"band": {
|
|
"p2_5": -0.0625,
|
|
"p50": 0.03125,
|
|
"p97_5": 0.15625
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 29,
|
|
"negative": 1
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": -15.71875,
|
|
"band": {
|
|
"p2_5": -57.627343749999994,
|
|
"p50": -15.34375,
|
|
"p97_5": 24.93828124999999
|
|
},
|
|
"directions": {
|
|
"positive": 15,
|
|
"zero": 0,
|
|
"negative": 17
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 0,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 13.25,
|
|
"median": 10,
|
|
"min": 0,
|
|
"max": 65
|
|
}
|
|
}
|
|
},
|
|
"period_at_s1": {
|
|
"left": "s1_eos",
|
|
"right": "s1_period",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": -0.125,
|
|
"band": {
|
|
"p2_5": -0.34375,
|
|
"p50": -0.125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 4,
|
|
"zero": 20,
|
|
"negative": 8
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 12,
|
|
"fail_to_pass": 4,
|
|
"pass_to_fail": 8,
|
|
"pass_to_pass": 8
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": -0.125,
|
|
"band": {
|
|
"p2_5": -0.34375,
|
|
"p50": -0.125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 4,
|
|
"zero": 20,
|
|
"negative": 8
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 12,
|
|
"fail_to_pass": 4,
|
|
"pass_to_fail": 8,
|
|
"pass_to_pass": 8
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0.03125,
|
|
"band": {
|
|
"p2_5": -0.0625,
|
|
"p50": 0.03125,
|
|
"p97_5": 0.125
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 29,
|
|
"negative": 1
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": -125.4375,
|
|
"band": {
|
|
"p2_5": -165.03125,
|
|
"p50": -125.71875,
|
|
"p97_5": -85.125
|
|
},
|
|
"directions": {
|
|
"positive": 4,
|
|
"zero": 0,
|
|
"negative": 28
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 0,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 3.6875,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 32
|
|
}
|
|
}
|
|
},
|
|
"system_at_eos": {
|
|
"left": "s0_eos",
|
|
"right": "s1_eos",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": -0.125,
|
|
"p50": 0,
|
|
"p97_5": 0.125
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 28,
|
|
"negative": 2
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 14,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 2,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": -0.125,
|
|
"p50": 0,
|
|
"p97_5": 0.125
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 28,
|
|
"negative": 2
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 14,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 2,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": 0,
|
|
"p50": 0,
|
|
"p97_5": 0
|
|
},
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 32,
|
|
"negative": 0
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": -21.15625,
|
|
"band": {
|
|
"p2_5": -47.846093749999994,
|
|
"p50": -20.78125,
|
|
"p97_5": 2.4382812499999886
|
|
},
|
|
"directions": {
|
|
"positive": 11,
|
|
"zero": 2,
|
|
"negative": 19
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 0,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 23.90625,
|
|
"median": 15,
|
|
"min": 0,
|
|
"max": 107
|
|
}
|
|
}
|
|
},
|
|
"system_at_period": {
|
|
"left": "s0_period",
|
|
"right": "s1_period",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": -0.09375,
|
|
"band": {
|
|
"p2_5": -0.25,
|
|
"p50": -0.09375,
|
|
"p97_5": 0.0625
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 25,
|
|
"negative": 5
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 15,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 5,
|
|
"pass_to_pass": 10
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": -0.09375,
|
|
"band": {
|
|
"p2_5": -0.25,
|
|
"p50": -0.09375,
|
|
"p97_5": 0.0625
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 25,
|
|
"negative": 5
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 15,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 5,
|
|
"pass_to_pass": 10
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": -0.09375,
|
|
"p50": 0,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 30,
|
|
"negative": 1
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": -130.875,
|
|
"band": {
|
|
"p2_5": -178.6875,
|
|
"p50": -131.28125,
|
|
"p97_5": -83.21796875000001
|
|
},
|
|
"directions": {
|
|
"positive": 6,
|
|
"zero": 1,
|
|
"negative": 25
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 1,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 11.875,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 109
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"math": {
|
|
"period_at_s0": {
|
|
"left": "s0_eos",
|
|
"right": "s0_period",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": -0.0625,
|
|
"band": {
|
|
"p2_5": -0.21875,
|
|
"p50": -0.0625,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 3,
|
|
"zero": 24,
|
|
"negative": 5
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 10,
|
|
"fail_to_pass": 3,
|
|
"pass_to_fail": 5,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": -0.0625,
|
|
"band": {
|
|
"p2_5": -0.21875,
|
|
"p50": -0.0625,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 3,
|
|
"zero": 24,
|
|
"negative": 5
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 10,
|
|
"fail_to_pass": 3,
|
|
"pass_to_fail": 5,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0.03125,
|
|
"band": {
|
|
"p2_5": 0,
|
|
"p50": 0.03125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 31,
|
|
"negative": 0
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": -8.21875,
|
|
"band": {
|
|
"p2_5": -27.56328125,
|
|
"p50": -7.8125,
|
|
"p97_5": 8.6875
|
|
},
|
|
"directions": {
|
|
"positive": 10,
|
|
"zero": 10,
|
|
"negative": 12
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 6,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 52.53125,
|
|
"median": 43,
|
|
"min": 0,
|
|
"max": 254
|
|
}
|
|
}
|
|
},
|
|
"period_at_s1": {
|
|
"left": "s1_eos",
|
|
"right": "s1_period",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": -0.09375,
|
|
"band": {
|
|
"p2_5": -0.25,
|
|
"p50": -0.09375,
|
|
"p97_5": 0.0625
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 25,
|
|
"negative": 5
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 11,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 5,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": -0.09375,
|
|
"band": {
|
|
"p2_5": -0.25,
|
|
"p50": -0.09375,
|
|
"p97_5": 0.0625
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 25,
|
|
"negative": 5
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 11,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 5,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": 0,
|
|
"p50": 0,
|
|
"p97_5": 0
|
|
},
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 32,
|
|
"negative": 0
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": 7.34375,
|
|
"band": {
|
|
"p2_5": -9.40703125,
|
|
"p50": 7.140625,
|
|
"p97_5": 26.09375
|
|
},
|
|
"directions": {
|
|
"positive": 15,
|
|
"zero": 1,
|
|
"negative": 16
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 1,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 22.90625,
|
|
"median": 8.5,
|
|
"min": 0,
|
|
"max": 152
|
|
}
|
|
}
|
|
},
|
|
"system_at_eos": {
|
|
"left": "s0_eos",
|
|
"right": "s1_eos",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": -0.15625,
|
|
"p50": 0,
|
|
"p97_5": 0.15625
|
|
},
|
|
"directions": {
|
|
"positive": 3,
|
|
"zero": 26,
|
|
"negative": 3
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 10,
|
|
"fail_to_pass": 3,
|
|
"pass_to_fail": 3,
|
|
"pass_to_pass": 16
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": -0.15625,
|
|
"p50": 0,
|
|
"p97_5": 0.15625
|
|
},
|
|
"directions": {
|
|
"positive": 3,
|
|
"zero": 26,
|
|
"negative": 3
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 10,
|
|
"fail_to_pass": 3,
|
|
"pass_to_fail": 3,
|
|
"pass_to_pass": 16
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0.03125,
|
|
"band": {
|
|
"p2_5": 0,
|
|
"p50": 0.03125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 1,
|
|
"zero": 31,
|
|
"negative": 0
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": 9.5625,
|
|
"band": {
|
|
"p2_5": -4.9375,
|
|
"p50": 9.234375,
|
|
"p97_5": 25.1875
|
|
},
|
|
"directions": {
|
|
"positive": 15,
|
|
"zero": 8,
|
|
"negative": 9
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 6,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 59.4375,
|
|
"median": 44.5,
|
|
"min": 0,
|
|
"max": 213
|
|
}
|
|
}
|
|
},
|
|
"system_at_period": {
|
|
"left": "s0_period",
|
|
"right": "s1_period",
|
|
"metrics": {
|
|
"fixed_budget_success": {
|
|
"point": -0.03125,
|
|
"band": {
|
|
"p2_5": -0.15625,
|
|
"p50": -0.03125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 27,
|
|
"negative": 3
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 13,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 3,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"strict_complete_success": {
|
|
"point": -0.03125,
|
|
"band": {
|
|
"p2_5": -0.15625,
|
|
"p50": -0.03125,
|
|
"p97_5": 0.09375
|
|
},
|
|
"directions": {
|
|
"positive": 2,
|
|
"zero": 27,
|
|
"negative": 3
|
|
},
|
|
"transition": {
|
|
"fail_to_fail": 13,
|
|
"fail_to_pass": 2,
|
|
"pass_to_fail": 3,
|
|
"pass_to_pass": 14
|
|
}
|
|
},
|
|
"natural_eos": {
|
|
"point": 0,
|
|
"band": {
|
|
"p2_5": 0,
|
|
"p50": 0,
|
|
"p97_5": 0
|
|
},
|
|
"directions": {
|
|
"positive": 0,
|
|
"zero": 32,
|
|
"negative": 0
|
|
},
|
|
"transition": null
|
|
},
|
|
"generated_tokens": {
|
|
"point": 25.125,
|
|
"band": {
|
|
"p2_5": 7.874218750000001,
|
|
"p50": 24.84375,
|
|
"p97_5": 44.53125
|
|
},
|
|
"directions": {
|
|
"positive": 18,
|
|
"zero": 2,
|
|
"negative": 12
|
|
},
|
|
"transition": null
|
|
}
|
|
},
|
|
"trajectory": {
|
|
"sources": 32,
|
|
"sharedUniformExact": 32,
|
|
"exactTrajectories": 1,
|
|
"commonPrefixTokens": {
|
|
"count": 32,
|
|
"mean": 27.75,
|
|
"median": 9.5,
|
|
"min": 0,
|
|
"max": 152
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"diagnostic": {
|
|
"code": {
|
|
"sourceIds": [
|
|
"HumanEval/31",
|
|
"HumanEval/22",
|
|
"HumanEval/53",
|
|
"HumanEval/141"
|
|
],
|
|
"tapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
],
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"left": "s0_eos",
|
|
"right": "s0_period",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": 0,
|
|
"T1": 0,
|
|
"T2": 0,
|
|
"T3": 0
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 0
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 0
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
26,
|
|
59,
|
|
13,
|
|
-51
|
|
],
|
|
[
|
|
49,
|
|
-7,
|
|
-38,
|
|
49
|
|
],
|
|
[
|
|
-48,
|
|
42,
|
|
-58,
|
|
-41
|
|
],
|
|
[
|
|
-85,
|
|
84,
|
|
137,
|
|
-24
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -14.5,
|
|
"T1": 44.5,
|
|
"T2": 13.5,
|
|
"T3": -16.75
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 130,
|
|
"median": 117,
|
|
"min": 91,
|
|
"max": 195
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 129.75,
|
|
"median": 105,
|
|
"min": 87,
|
|
"max": 222
|
|
}
|
|
}
|
|
},
|
|
"period_at_s1": {
|
|
"left": "s1_eos",
|
|
"right": "s1_period",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
-1,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -0.25,
|
|
"T1": 0,
|
|
"T2": 0,
|
|
"T3": 0
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 1
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
-285,
|
|
-249,
|
|
-222,
|
|
-266
|
|
],
|
|
[
|
|
22,
|
|
-35,
|
|
120,
|
|
-39
|
|
],
|
|
[
|
|
-121,
|
|
-70,
|
|
-5,
|
|
-100
|
|
],
|
|
[
|
|
-50,
|
|
-266,
|
|
23,
|
|
-248
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -108.5,
|
|
"T1": -155,
|
|
"T2": -21,
|
|
"T3": -163.25
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 276.75,
|
|
"median": 269,
|
|
"min": 227,
|
|
"max": 342
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 156.75,
|
|
"median": 137.5,
|
|
"min": 63,
|
|
"max": 289
|
|
}
|
|
}
|
|
},
|
|
"system_at_eos": {
|
|
"left": "s0_eos",
|
|
"right": "s1_eos",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": 0,
|
|
"T1": 0,
|
|
"T2": 0,
|
|
"T3": 0
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 0
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 0,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 0
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
-50,
|
|
-64,
|
|
-77,
|
|
-113
|
|
],
|
|
[
|
|
26,
|
|
13,
|
|
-154,
|
|
22
|
|
],
|
|
[
|
|
-6,
|
|
-9,
|
|
-58,
|
|
-48
|
|
],
|
|
[
|
|
-206,
|
|
84,
|
|
-71,
|
|
-19
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -59,
|
|
"T1": 6,
|
|
"T2": -90,
|
|
"T3": -39.5
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 152.75,
|
|
"median": 141.5,
|
|
"min": 96,
|
|
"max": 232
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 146.25,
|
|
"median": 121.5,
|
|
"min": 52,
|
|
"max": 290
|
|
}
|
|
}
|
|
},
|
|
"system_at_period": {
|
|
"left": "s0_period",
|
|
"right": "s1_period",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
-1,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -0.25,
|
|
"T1": 0,
|
|
"T2": 0,
|
|
"T3": 0
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 0.25,
|
|
"median": 0,
|
|
"min": 0,
|
|
"max": 1
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
-361,
|
|
-372,
|
|
-312,
|
|
-328
|
|
],
|
|
[
|
|
-1,
|
|
-15,
|
|
4,
|
|
-66
|
|
],
|
|
[
|
|
-79,
|
|
-121,
|
|
-5,
|
|
-107
|
|
],
|
|
[
|
|
-171,
|
|
-266,
|
|
-185,
|
|
-243
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -153,
|
|
"T1": -193.5,
|
|
"T2": -124.5,
|
|
"T3": -186
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 323.75,
|
|
"median": 336.5,
|
|
"min": 262,
|
|
"max": 360
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 85.25,
|
|
"median": 82.5,
|
|
"min": 60,
|
|
"max": 116
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"math": {
|
|
"sourceIds": [
|
|
"gsm8k/test/1069",
|
|
"gsm8k/test/0546",
|
|
"gsm8k/test/1050",
|
|
"gsm8k/test/0584"
|
|
],
|
|
"tapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
],
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"left": "s0_eos",
|
|
"right": "s0_period",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
-1,
|
|
0,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
0,
|
|
-1,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": 0,
|
|
"T1": 0,
|
|
"T2": -0.25,
|
|
"T3": 0.25
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1,
|
|
"min": 0,
|
|
"max": 2
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1,
|
|
"min": 0,
|
|
"max": 2
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
48,
|
|
-19,
|
|
0
|
|
],
|
|
[
|
|
-13,
|
|
-9,
|
|
0,
|
|
8
|
|
],
|
|
[
|
|
0,
|
|
35,
|
|
-1,
|
|
0
|
|
],
|
|
[
|
|
-8,
|
|
51,
|
|
-20,
|
|
8
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -5.25,
|
|
"T1": 31.25,
|
|
"T2": -10,
|
|
"T3": 4
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 25.25,
|
|
"median": 16.5,
|
|
"min": 8,
|
|
"max": 60
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 48.75,
|
|
"median": 51.5,
|
|
"min": 21,
|
|
"max": 71
|
|
}
|
|
}
|
|
},
|
|
"period_at_s1": {
|
|
"left": "s1_eos",
|
|
"right": "s1_period",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
-1,
|
|
0,
|
|
-1
|
|
],
|
|
[
|
|
-1,
|
|
-1,
|
|
-1,
|
|
-1
|
|
],
|
|
[
|
|
-1,
|
|
-1,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -0.5,
|
|
"T1": -0.75,
|
|
"T2": -0.25,
|
|
"T3": -0.5
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 0.5,
|
|
"median": 0.5,
|
|
"min": 0,
|
|
"max": 1
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
16,
|
|
-6,
|
|
30,
|
|
-17
|
|
],
|
|
[
|
|
13,
|
|
-25,
|
|
-27,
|
|
-44
|
|
],
|
|
[
|
|
3,
|
|
67,
|
|
40,
|
|
106
|
|
],
|
|
[
|
|
-30,
|
|
-35,
|
|
-30,
|
|
-55
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": 0.5,
|
|
"T1": 0.25,
|
|
"T2": 3.25,
|
|
"T3": -2.5
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 94.75,
|
|
"median": 86,
|
|
"min": 46,
|
|
"max": 161
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 58,
|
|
"median": 52,
|
|
"min": 25,
|
|
"max": 103
|
|
}
|
|
}
|
|
},
|
|
"system_at_eos": {
|
|
"left": "s0_eos",
|
|
"right": "s1_eos",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
-1,
|
|
1,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
1,
|
|
-1,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": 0,
|
|
"T1": 0.5,
|
|
"T2": -0.25,
|
|
"T3": 0.25
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 1.25,
|
|
"median": 1,
|
|
"min": 1,
|
|
"max": 2
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1,
|
|
"min": 0,
|
|
"max": 2
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
-36,
|
|
64,
|
|
-17,
|
|
-3
|
|
],
|
|
[
|
|
-26,
|
|
24,
|
|
0,
|
|
8
|
|
],
|
|
[
|
|
34,
|
|
37,
|
|
-1,
|
|
0
|
|
],
|
|
[
|
|
16,
|
|
94,
|
|
-12,
|
|
126
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -3,
|
|
"T1": 54.75,
|
|
"T2": -7.5,
|
|
"T3": 32.75
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 71.5,
|
|
"median": 70,
|
|
"min": 17,
|
|
"max": 129
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 81.5,
|
|
"median": 75,
|
|
"min": 38,
|
|
"max": 138
|
|
}
|
|
}
|
|
},
|
|
"system_at_period": {
|
|
"left": "s0_period",
|
|
"right": "s1_period",
|
|
"success": {
|
|
"matrix": [
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0,
|
|
-1
|
|
],
|
|
[
|
|
-1,
|
|
-1,
|
|
-1,
|
|
-1
|
|
],
|
|
[
|
|
-1,
|
|
0,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": -0.5,
|
|
"T1": -0.25,
|
|
"T2": -0.25,
|
|
"T3": -0.5
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 1,
|
|
"median": 1,
|
|
"min": 1,
|
|
"max": 1
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 0.5,
|
|
"median": 0.5,
|
|
"min": 0,
|
|
"max": 1
|
|
}
|
|
},
|
|
"tokens": {
|
|
"matrix": [
|
|
[
|
|
-20,
|
|
10,
|
|
32,
|
|
-20
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
-27,
|
|
-44
|
|
],
|
|
[
|
|
37,
|
|
69,
|
|
40,
|
|
106
|
|
],
|
|
[
|
|
-6,
|
|
8,
|
|
-22,
|
|
63
|
|
]
|
|
],
|
|
"tapeMeans": {
|
|
"T0": 2.75,
|
|
"T1": 23.75,
|
|
"T2": 5.75,
|
|
"T3": 26.25
|
|
},
|
|
"taskRangeWithinTape": {
|
|
"count": 4,
|
|
"mean": 83.75,
|
|
"median": 64,
|
|
"min": 57,
|
|
"max": 150
|
|
},
|
|
"tapeRangeWithinTask": {
|
|
"count": 4,
|
|
"mean": 64.5,
|
|
"median": 60.5,
|
|
"min": 52,
|
|
"max": 85
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"tasks": {
|
|
"code": [
|
|
{
|
|
"id": "HumanEval/31",
|
|
"index": 0,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 394,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "9bf7b1d42c68",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
262,
|
|
62,
|
|
6620,
|
|
63
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 344,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b2c8363abfa6",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
245,
|
|
1604,
|
|
317,
|
|
9966,
|
|
11
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 420,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4983d736318e",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
262,
|
|
62,
|
|
6620,
|
|
63
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 59,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5cc9c902a87c",
|
|
"firstTokenIds": [
|
|
977,
|
|
317,
|
|
62,
|
|
6620,
|
|
7,
|
|
77,
|
|
1780,
|
|
185
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 26,
|
|
"commonPrefixTokens": 27,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -285,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -50,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -361,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/44",
|
|
"index": 1,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 496,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "19e1162b6f49",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
4874,
|
|
62,
|
|
4915,
|
|
63
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 445,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5f0d89d2123c",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
543,
|
|
2606
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 498,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e23ca257f456",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
543,
|
|
938
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 180,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6b5783005111",
|
|
"firstTokenIds": [
|
|
977,
|
|
2262,
|
|
62,
|
|
4915,
|
|
7,
|
|
87,
|
|
25,
|
|
1098
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 2,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -265,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -51,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -318,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/133",
|
|
"index": 2,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 360,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "02a4a5a394bb",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 357,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1f3227dedff6",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 352,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "657c65932b67",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 78,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "bf4fa0912d02",
|
|
"firstTokenIds": [
|
|
977,
|
|
2555,
|
|
62,
|
|
72582,
|
|
7,
|
|
32808,
|
|
1780,
|
|
185
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -8,
|
|
"commonPrefixTokens": 25,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -279,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -3,
|
|
"commonPrefixTokens": 25,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -274,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/23",
|
|
"index": 3,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 97,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "fdda57e06407",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 101,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "07b6c2b6a7de",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 327,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "927ab37c7449",
|
|
"firstTokenIds": [
|
|
429,
|
|
1157,
|
|
2030,
|
|
73675,
|
|
63,
|
|
317,
|
|
4218,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 48,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f8eebd031ad0",
|
|
"firstTokenIds": [
|
|
977,
|
|
56958,
|
|
7,
|
|
2612,
|
|
25,
|
|
1406,
|
|
8,
|
|
5052
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 230,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -53,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 4,
|
|
"commonPrefixTokens": 11,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -279,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/147",
|
|
"index": 4,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 287,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "7968d31e98e1",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 392,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6d2b0c45c73e",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 442,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a7ffd6b8c686",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "ca6207e139f1",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 155,
|
|
"commonPrefixTokens": 16,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 120,
|
|
"commonPrefixTokens": 32,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 105,
|
|
"commonPrefixTokens": 16,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 70,
|
|
"commonPrefixTokens": 31,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/34",
|
|
"index": 5,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 207,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e8306211b557",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
520,
|
|
2066
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 173,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "efbee579549f",
|
|
"firstTokenIds": [
|
|
2158,
|
|
2606,
|
|
245,
|
|
1157,
|
|
2030,
|
|
19380,
|
|
63,
|
|
344
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 192,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2614cbbf28ab",
|
|
"firstTokenIds": [
|
|
2158,
|
|
2606,
|
|
245,
|
|
1157,
|
|
2030,
|
|
19380,
|
|
63,
|
|
344
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 17,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c52008cb9c22",
|
|
"firstTokenIds": [
|
|
977,
|
|
4730,
|
|
7,
|
|
75,
|
|
25,
|
|
1525,
|
|
1780,
|
|
185
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -15,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -156,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -34,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -175,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/123",
|
|
"index": 6,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 331,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "349132c5055e",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
543,
|
|
4353
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 329,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "92b0903aff50",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
543,
|
|
4353
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 507,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e03d0688f20a",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
543,
|
|
4353
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 205,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "92cce4ae0633",
|
|
"firstTokenIds": [
|
|
977,
|
|
752,
|
|
62,
|
|
16751,
|
|
62,
|
|
1316,
|
|
10243,
|
|
89
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 176,
|
|
"commonPrefixTokens": 25,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -124,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -2,
|
|
"commonPrefixTokens": 33,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -302,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/77",
|
|
"index": 7,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "extract_failed",
|
|
"tokens": 278,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4a62750de07e",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
274,
|
|
10858,
|
|
2030,
|
|
64,
|
|
63
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 303,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "09fa0110978d",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
245,
|
|
2028,
|
|
10858,
|
|
2030,
|
|
64
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 439,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cf801d7b12e4",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
245,
|
|
2028,
|
|
10858,
|
|
2030,
|
|
64
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 112,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "488b24d54f89",
|
|
"firstTokenIds": [
|
|
977,
|
|
317,
|
|
48808,
|
|
7,
|
|
64,
|
|
1780,
|
|
185,
|
|
300
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 161,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -191,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 25,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -327,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/22",
|
|
"index": 8,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 163,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "30cf3dc459e6",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030,
|
|
8664
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 189,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b0d6a6b53629",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
245,
|
|
12974,
|
|
1157,
|
|
344,
|
|
16674
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 212,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6faf844ccf7c",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
8664,
|
|
62,
|
|
6607,
|
|
8563
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 211,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ed3527877810",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6226,
|
|
245,
|
|
1525,
|
|
280,
|
|
688,
|
|
12974,
|
|
3039
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 49,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 22,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 26,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -1,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/127",
|
|
"index": 9,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 475,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ba2bceaaa413",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 468,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2bd9409092aa",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 439,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "61faa4d92f7e",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 426,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "9e89f0b5ea23",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -36,
|
|
"commonPrefixTokens": 16,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -42,
|
|
"commonPrefixTokens": 16,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -7,
|
|
"commonPrefixTokens": 22,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -13,
|
|
"commonPrefixTokens": 58,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/37",
|
|
"index": 10,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 336,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "7f36b1f2ee49",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
2606
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 345,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "94a692a59865",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
2606
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 332,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "98e50a22c918",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
2606
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 117,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1dde7533a323",
|
|
"firstTokenIds": [
|
|
977,
|
|
3734,
|
|
62,
|
|
8589,
|
|
7,
|
|
75,
|
|
25,
|
|
1525
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -4,
|
|
"commonPrefixTokens": 14,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -228,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 9,
|
|
"commonPrefixTokens": 14,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -215,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/48",
|
|
"index": 11,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 165,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1f279a162ceb",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 163,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "655d2ec6e291",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 310,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "25de077ab6e7",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
12974,
|
|
1157,
|
|
2030,
|
|
262
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 25,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cf639cf871c2",
|
|
"firstTokenIds": [
|
|
977,
|
|
317,
|
|
62,
|
|
16435,
|
|
515,
|
|
7288,
|
|
7,
|
|
822
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 145,
|
|
"commonPrefixTokens": 4,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -138,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -2,
|
|
"commonPrefixTokens": 105,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -285,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/158",
|
|
"index": 12,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 383,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ec2ca315c455",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 387,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "64d60661fc00",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 343,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b3fe45cbd93a",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 385,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5b5cdc874133",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -40,
|
|
"commonPrefixTokens": 29,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -2,
|
|
"commonPrefixTokens": 8,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 4,
|
|
"commonPrefixTokens": 41,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 42,
|
|
"commonPrefixTokens": 8,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/70",
|
|
"index": 13,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 429,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6775c6ea0d43",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
43744,
|
|
62,
|
|
14318,
|
|
62
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 342,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "804179fa4bc8",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
43744,
|
|
62,
|
|
14318,
|
|
62
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "syntax_error",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "1b599cc3a8aa",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
43744,
|
|
62,
|
|
14318,
|
|
62
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 440,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "05f2d4ff1eca",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
43744,
|
|
62,
|
|
14318,
|
|
62
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 83,
|
|
"commonPrefixTokens": 12,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 98,
|
|
"commonPrefixTokens": 22,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -87,
|
|
"commonPrefixTokens": 13,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -72,
|
|
"commonPrefixTokens": 12,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/89",
|
|
"index": 14,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 323,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "20cf3156bfba",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 336,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b49e265f4d94",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 166,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "25ca4d63b9f3",
|
|
"firstTokenIds": [
|
|
977,
|
|
39123,
|
|
7,
|
|
82,
|
|
1780,
|
|
185,
|
|
300,
|
|
23504
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 144,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "3e1eb21230e3",
|
|
"firstTokenIds": [
|
|
977,
|
|
39123,
|
|
7,
|
|
82,
|
|
1780,
|
|
185,
|
|
300,
|
|
23504
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -157,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -192,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 1,
|
|
"tokenDelta": 13,
|
|
"commonPrefixTokens": 21,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -22,
|
|
"commonPrefixTokens": 31,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/9",
|
|
"index": 15,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 171,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "639217e488ee",
|
|
"firstTokenIds": [
|
|
473,
|
|
23134,
|
|
1666,
|
|
5896,
|
|
11,
|
|
68776,
|
|
185,
|
|
185
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 267,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "91444ab53c19",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
481,
|
|
938
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 109,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a5f244910716",
|
|
"firstTokenIds": [
|
|
473,
|
|
23134,
|
|
1666,
|
|
5896,
|
|
11,
|
|
68776,
|
|
185,
|
|
185
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 109,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a5f244910716",
|
|
"firstTokenIds": [
|
|
473,
|
|
23134,
|
|
1666,
|
|
5896,
|
|
11,
|
|
68776,
|
|
185,
|
|
185
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -62,
|
|
"commonPrefixTokens": 8,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -158,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 96,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 109,
|
|
"exactTrajectory": true
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/53",
|
|
"index": 16,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 212,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "bf812ffe1064",
|
|
"firstTokenIds": [
|
|
429,
|
|
1157,
|
|
2030,
|
|
1770,
|
|
7,
|
|
87,
|
|
25,
|
|
1098
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 206,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "008f3ad4a12b",
|
|
"firstTokenIds": [
|
|
429,
|
|
1157,
|
|
6525,
|
|
340,
|
|
4286,
|
|
317,
|
|
29444,
|
|
13
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 164,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "08f88dd5e77e",
|
|
"firstTokenIds": [
|
|
429,
|
|
2985,
|
|
340,
|
|
4286,
|
|
317,
|
|
245,
|
|
1157,
|
|
6525
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "extract_failed",
|
|
"tokens": 85,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "94bb19f2316d",
|
|
"firstTokenIds": [
|
|
429,
|
|
1157,
|
|
2030,
|
|
1770,
|
|
63,
|
|
4497,
|
|
984,
|
|
4823
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -48,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -121,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -6,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -79,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/145",
|
|
"index": 17,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 406,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4def57bad977",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
2862,
|
|
62,
|
|
1956,
|
|
62
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 398,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2906305486ea",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
2862,
|
|
62,
|
|
1956,
|
|
62
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 346,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ad307492cbf4",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 270,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2136be2d0e53",
|
|
"firstTokenIds": [
|
|
977,
|
|
1835,
|
|
62,
|
|
1956,
|
|
62,
|
|
12140,
|
|
7,
|
|
6485
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -60,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -128,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -8,
|
|
"commonPrefixTokens": 51,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -76,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/35",
|
|
"index": 18,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 165,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "bde367e7e902",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 222,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d5698b92bf4f",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
3978,
|
|
62,
|
|
7633,
|
|
63
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 174,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d2d65b7ad848",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
938
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 17,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "30176e3a09e7",
|
|
"firstTokenIds": [
|
|
977,
|
|
3044,
|
|
62,
|
|
7633,
|
|
7,
|
|
75,
|
|
25,
|
|
1525
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 9,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -205,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 57,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -157,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/103",
|
|
"index": 19,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 416,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ab0679e0aa86",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 179,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "134c7f287a3b",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 281,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b0315b03c0b9",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 47,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cca280f7a5e1",
|
|
"firstTokenIds": [
|
|
977,
|
|
27840,
|
|
62,
|
|
36744,
|
|
7,
|
|
77,
|
|
11,
|
|
273
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -135,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -132,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -237,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -234,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/25",
|
|
"index": 20,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 404,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "7dfc3c99b18d",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
24511,
|
|
945,
|
|
63,
|
|
1157
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 465,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f51a0458d315",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
24511,
|
|
945,
|
|
63,
|
|
1157
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 425,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5dea318510b7",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
24511,
|
|
945,
|
|
63,
|
|
1157
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 208,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e8c5dbc3ee98",
|
|
"firstTokenIds": [
|
|
977,
|
|
6088,
|
|
945,
|
|
7,
|
|
77,
|
|
25,
|
|
1098,
|
|
8
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 21,
|
|
"commonPrefixTokens": 34,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -257,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 61,
|
|
"commonPrefixTokens": 20,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -217,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/98",
|
|
"index": 21,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 254,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "09f519a955e2",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 239,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "165669786bde",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 260,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "20e180c35f11",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
245,
|
|
12974,
|
|
1157,
|
|
344,
|
|
16839
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 67,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "87962fdd3a88",
|
|
"firstTokenIds": [
|
|
977,
|
|
1690,
|
|
62,
|
|
22352,
|
|
7,
|
|
82,
|
|
1780,
|
|
185
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 6,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -172,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -15,
|
|
"commonPrefixTokens": 50,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -193,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/115",
|
|
"index": 22,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "runtime_error",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "19ac18a0f9d1",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
254,
|
|
2066,
|
|
280,
|
|
36525,
|
|
4627,
|
|
45176
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "98781d758100",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 243,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "13109dcb583b",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 456,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f0851a034394",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -269,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -56,
|
|
"commonPrefixTokens": 23,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 213,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/10",
|
|
"index": 23,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "timeout",
|
|
"tokens": 206,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "98d9783e6b56",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030,
|
|
7690
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "timeout",
|
|
"tokens": 213,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8cbfee1a8652",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030,
|
|
7690
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "timeout",
|
|
"tokens": 208,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "90b861364699",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
2030,
|
|
7690
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 92,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f128629f477e",
|
|
"firstTokenIds": [
|
|
977,
|
|
1099,
|
|
62,
|
|
16435,
|
|
515,
|
|
7288,
|
|
7,
|
|
2612
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 2,
|
|
"commonPrefixTokens": 65,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -121,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 7,
|
|
"commonPrefixTokens": 107,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -116,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/141",
|
|
"index": 24,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 496,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "68de6956bb45",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
2810,
|
|
62,
|
|
1531,
|
|
62
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 290,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "3511784c6023",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
245,
|
|
12974,
|
|
1157,
|
|
344,
|
|
14180
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 411,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d8c4fa6913a3",
|
|
"firstTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
2810,
|
|
62,
|
|
1531,
|
|
62
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 240,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "7d66767a0af1",
|
|
"firstTokenIds": [
|
|
977,
|
|
1761,
|
|
62,
|
|
1531,
|
|
62,
|
|
4897,
|
|
7,
|
|
2810
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -85,
|
|
"commonPrefixTokens": 28,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -50,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -206,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -171,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/91",
|
|
"index": 25,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 449,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ef4c79e23886",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
938
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 316,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b5a6c981027a",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
938
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 267,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f3aa2c04d8a6",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
938
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 269,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1c4b76f6c982",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
340,
|
|
481,
|
|
938
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -182,
|
|
"commonPrefixTokens": 23,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -47,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -133,
|
|
"commonPrefixTokens": 23,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 2,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/122",
|
|
"index": 26,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 163,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a72564638ef4",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 157,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e044667fa7e1",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 193,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "48ea0f155d34",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 52,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2e80616effd3",
|
|
"firstTokenIds": [
|
|
977,
|
|
962,
|
|
62,
|
|
22208,
|
|
7,
|
|
3049,
|
|
11,
|
|
530
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 1,
|
|
"tokenDelta": 30,
|
|
"commonPrefixTokens": 25,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -105,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -6,
|
|
"commonPrefixTokens": 62,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -141,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/75",
|
|
"index": 27,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "fafe77847176",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
245,
|
|
2028,
|
|
1604,
|
|
317,
|
|
254
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "7baf64d1d00b",
|
|
"firstTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
245,
|
|
2028,
|
|
1604,
|
|
317,
|
|
254
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 182,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "bbba56878e09",
|
|
"firstTokenIds": [
|
|
977,
|
|
317,
|
|
62,
|
|
84024,
|
|
62,
|
|
6620,
|
|
7,
|
|
64
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 234,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "30dfdd28a88a",
|
|
"firstTokenIds": [
|
|
977,
|
|
317,
|
|
62,
|
|
84024,
|
|
62,
|
|
6620,
|
|
7,
|
|
64
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -330,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -278,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 73,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 52,
|
|
"commonPrefixTokens": 99,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/134",
|
|
"index": 28,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 390,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c81cab156949",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
9674,
|
|
280,
|
|
254,
|
|
1157
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 248,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "30185955df38",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
12974,
|
|
2985,
|
|
327,
|
|
254
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 347,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a661269c6c22",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 365,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e14af9de6150",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -43,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 117,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -142,
|
|
"commonPrefixTokens": 4,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 18,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/142",
|
|
"index": 29,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 481,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ad25d29cc918",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 448,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e9fbc617db8a",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 433,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ffe15a49500a",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 137,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f68000dbffc5",
|
|
"firstTokenIds": [
|
|
977,
|
|
2555,
|
|
62,
|
|
72582,
|
|
7,
|
|
32808,
|
|
1780,
|
|
185
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -48,
|
|
"commonPrefixTokens": 17,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -311,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -33,
|
|
"commonPrefixTokens": 45,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -296,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/149",
|
|
"index": 30,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 388,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1099cc41e2b4",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 348,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "52f9a00b95da",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 289,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "214637cf12a9",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 132,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "78a43f2715ed",
|
|
"firstTokenIds": [
|
|
977,
|
|
24350,
|
|
62,
|
|
2502,
|
|
62,
|
|
1805,
|
|
7,
|
|
32808
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -99,
|
|
"commonPrefixTokens": 18,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -216,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -40,
|
|
"commonPrefixTokens": 18,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -157,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "HumanEval/102",
|
|
"index": 31,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 214,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d447e53cf6b5",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 192,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ddd4e52d97eb",
|
|
"firstTokenIds": [
|
|
4462,
|
|
6,
|
|
82,
|
|
254,
|
|
12974,
|
|
2985,
|
|
327,
|
|
254
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 237,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4bfba503871a",
|
|
"firstTokenIds": [
|
|
4462,
|
|
317,
|
|
245,
|
|
12974,
|
|
3418,
|
|
327,
|
|
254,
|
|
2066
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "assertion_failed",
|
|
"tokens": 133,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e4876c9b1fe1",
|
|
"firstTokenIds": [
|
|
977,
|
|
4352,
|
|
62,
|
|
6485,
|
|
7,
|
|
87,
|
|
11,
|
|
320
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 23,
|
|
"commonPrefixTokens": 29,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -59,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -22,
|
|
"commonPrefixTokens": 1,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -104,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
}
|
|
],
|
|
"math": [
|
|
{
|
|
"id": "gsm8k/test/1069",
|
|
"index": 0,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 206,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cbf1a78262d6",
|
|
"firstTokenIds": [
|
|
2195,
|
|
1063,
|
|
60522,
|
|
20570,
|
|
56543,
|
|
207,
|
|
17,
|
|
15
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 170,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "830a30d938d1",
|
|
"firstTokenIds": [
|
|
2195,
|
|
1063,
|
|
60522,
|
|
20570,
|
|
56543,
|
|
207,
|
|
17,
|
|
15
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 206,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c2faeaca9920",
|
|
"firstTokenIds": [
|
|
2195,
|
|
1063,
|
|
60522,
|
|
20570,
|
|
56543,
|
|
207,
|
|
17,
|
|
15
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 186,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "18792fc3b466",
|
|
"firstTokenIds": [
|
|
2195,
|
|
1063,
|
|
60522,
|
|
20570,
|
|
56543,
|
|
207,
|
|
17,
|
|
15
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 24,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 16,
|
|
"commonPrefixTokens": 53,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -36,
|
|
"commonPrefixTokens": 24,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -20,
|
|
"commonPrefixTokens": 53,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1228",
|
|
"index": 1,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 233,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2e48132c5b40",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
1604,
|
|
280,
|
|
27003
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 236,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c218feb5a172",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
1604,
|
|
280,
|
|
27003
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 264,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "378dae8e10af",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
1604,
|
|
280,
|
|
27003
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 270,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8ab456c76e1d",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
1604,
|
|
280,
|
|
27003
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 31,
|
|
"commonPrefixTokens": 94,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 34,
|
|
"commonPrefixTokens": 11,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 3,
|
|
"commonPrefixTokens": 172,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 6,
|
|
"commonPrefixTokens": 11,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0144",
|
|
"index": 2,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 193,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "201d42085f08",
|
|
"firstTokenIds": [
|
|
17654,
|
|
643,
|
|
207,
|
|
18,
|
|
6287,
|
|
6514,
|
|
285,
|
|
317
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 206,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b747a7676701",
|
|
"firstTokenIds": [
|
|
17654,
|
|
643,
|
|
207,
|
|
18,
|
|
6287,
|
|
6514,
|
|
285,
|
|
317
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 207,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1c98102f4c1e",
|
|
"firstTokenIds": [
|
|
17654,
|
|
643,
|
|
207,
|
|
18,
|
|
6287,
|
|
6514,
|
|
285,
|
|
317
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 229,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d955bcccd11e",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
11462,
|
|
254,
|
|
3223
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 14,
|
|
"commonPrefixTokens": 48,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 23,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 13,
|
|
"commonPrefixTokens": 48,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 22,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1251",
|
|
"index": 3,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 272,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "42e151391e6d",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 282,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "7a7a9679e49c",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 297,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "04248251fa86",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
2973,
|
|
1224,
|
|
254,
|
|
766,
|
|
359
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 280,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cb7ccb745dd5",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
933,
|
|
276
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 25,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -2,
|
|
"commonPrefixTokens": 8,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 10,
|
|
"commonPrefixTokens": 17,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -17,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0676",
|
|
"index": 4,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 168,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "628cc9980e57",
|
|
"firstTokenIds": [
|
|
53183,
|
|
6,
|
|
82,
|
|
9423,
|
|
317,
|
|
207,
|
|
17,
|
|
14
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 147,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2e79f872a9ef",
|
|
"firstTokenIds": [
|
|
53183,
|
|
6,
|
|
82,
|
|
9423,
|
|
317,
|
|
207,
|
|
17,
|
|
14
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 147,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "9b41a45d50a1",
|
|
"firstTokenIds": [
|
|
53183,
|
|
6,
|
|
82,
|
|
9423,
|
|
438,
|
|
207,
|
|
17,
|
|
14
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 140,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "dc3ff9e25291",
|
|
"firstTokenIds": [
|
|
53183,
|
|
6,
|
|
82,
|
|
9423,
|
|
438,
|
|
207,
|
|
17,
|
|
14
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -21,
|
|
"commonPrefixTokens": 4,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -7,
|
|
"commonPrefixTokens": 4,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -21,
|
|
"commonPrefixTokens": 32,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -7,
|
|
"commonPrefixTokens": 92,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0126",
|
|
"index": 5,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 72,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6f5a09833074",
|
|
"firstTokenIds": [
|
|
53183,
|
|
8939,
|
|
207,
|
|
18,
|
|
23,
|
|
14447,
|
|
98540,
|
|
13
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 72,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6f5a09833074",
|
|
"firstTokenIds": [
|
|
53183,
|
|
8939,
|
|
207,
|
|
18,
|
|
23,
|
|
14447,
|
|
98540,
|
|
13
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 72,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6f5a09833074",
|
|
"firstTokenIds": [
|
|
53183,
|
|
8939,
|
|
207,
|
|
18,
|
|
23,
|
|
14447,
|
|
98540,
|
|
13
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 69,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4513aa0b4ec7",
|
|
"firstTokenIds": [
|
|
53183,
|
|
8939,
|
|
207,
|
|
18,
|
|
23,
|
|
14447,
|
|
98540,
|
|
13
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 72,
|
|
"exactTrajectory": true
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -3,
|
|
"commonPrefixTokens": 61,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 72,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -3,
|
|
"commonPrefixTokens": 61,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0438",
|
|
"index": 6,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 359,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4496e6fd6e30",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 305,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "04fca59a3357",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 252,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d080eda4535c",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 245,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "aa46af5038f0",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -107,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -60,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -54,
|
|
"commonPrefixTokens": 32,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -7,
|
|
"commonPrefixTokens": 104,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1299",
|
|
"index": 7,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 111,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2fa8c2f0d99d",
|
|
"firstTokenIds": [
|
|
40804,
|
|
41504,
|
|
363,
|
|
18,
|
|
13,
|
|
20,
|
|
15,
|
|
331
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 107,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "bc76b9b932ad",
|
|
"firstTokenIds": [
|
|
40804,
|
|
41504,
|
|
363,
|
|
18,
|
|
13,
|
|
20,
|
|
15,
|
|
331
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 111,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1621f5c8b919",
|
|
"firstTokenIds": [
|
|
40804,
|
|
41504,
|
|
363,
|
|
18,
|
|
13,
|
|
20,
|
|
15,
|
|
331
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 106,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "57c5b323b6e0",
|
|
"firstTokenIds": [
|
|
40804,
|
|
41504,
|
|
363,
|
|
18,
|
|
13,
|
|
20,
|
|
15,
|
|
331
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 69,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -1,
|
|
"commonPrefixTokens": 37,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -4,
|
|
"commonPrefixTokens": 57,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -5,
|
|
"commonPrefixTokens": 37,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0546",
|
|
"index": 8,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 120,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "dc49a0944bb1",
|
|
"firstTokenIds": [
|
|
93435,
|
|
14870,
|
|
254,
|
|
32628,
|
|
279,
|
|
2201,
|
|
9682,
|
|
11
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 94,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c2793629ec5b",
|
|
"firstTokenIds": [
|
|
93435,
|
|
14870,
|
|
254,
|
|
32628,
|
|
279,
|
|
2201,
|
|
9682,
|
|
11
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 107,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b9ce7eb06d11",
|
|
"firstTokenIds": [
|
|
93435,
|
|
14870,
|
|
254,
|
|
32628,
|
|
9682,
|
|
11,
|
|
588,
|
|
317
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 107,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "71deb875dfce",
|
|
"firstTokenIds": [
|
|
93435,
|
|
14870,
|
|
254,
|
|
3912,
|
|
32628,
|
|
9682,
|
|
11,
|
|
588
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -13,
|
|
"commonPrefixTokens": 4,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 13,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -26,
|
|
"commonPrefixTokens": 34,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0537",
|
|
"index": 9,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 122,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "95723a6a422a",
|
|
"firstTokenIds": [
|
|
429,
|
|
1022,
|
|
2135,
|
|
317,
|
|
207,
|
|
18,
|
|
16,
|
|
11237
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 122,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "95723a6a422a",
|
|
"firstTokenIds": [
|
|
429,
|
|
1022,
|
|
2135,
|
|
317,
|
|
207,
|
|
18,
|
|
16,
|
|
11237
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 122,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "95723a6a422a",
|
|
"firstTokenIds": [
|
|
429,
|
|
1022,
|
|
2135,
|
|
317,
|
|
207,
|
|
18,
|
|
16,
|
|
11237
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 181,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "bcb6e4ca9af6",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
254,
|
|
3223,
|
|
22321,
|
|
280,
|
|
254,
|
|
2685
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 122,
|
|
"exactTrajectory": true
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 59,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 122,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 59,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0226",
|
|
"index": 10,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 191,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "85ab0782cea2",
|
|
"firstTokenIds": [
|
|
7518,
|
|
15920,
|
|
643,
|
|
207,
|
|
18,
|
|
15,
|
|
4043,
|
|
1449
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 328,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d4fe2f6566c5",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 310,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8ea8fd539f3d",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 251,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "eeebfcc26f88",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 119,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -77,
|
|
"commonPrefixTokens": 43,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 1,
|
|
"tokenDelta": 137,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -59,
|
|
"commonPrefixTokens": 43,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1128",
|
|
"index": 11,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 124,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "409230e40c0b",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
76282,
|
|
6,
|
|
82,
|
|
1648
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 126,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5ddd77291ff1",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
76282,
|
|
6,
|
|
82,
|
|
1648
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 128,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "012b66fdfbc3",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
76282,
|
|
6,
|
|
82,
|
|
1648
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 146,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "4b964e74637c",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
76282,
|
|
6,
|
|
82,
|
|
1648
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 4,
|
|
"commonPrefixTokens": 59,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 20,
|
|
"commonPrefixTokens": 19,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 2,
|
|
"commonPrefixTokens": 56,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 18,
|
|
"commonPrefixTokens": 19,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0760",
|
|
"index": 12,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 157,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "be807c966edf",
|
|
"firstTokenIds": [
|
|
207,
|
|
17,
|
|
15,
|
|
4,
|
|
280,
|
|
207,
|
|
20,
|
|
15
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 157,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "be807c966edf",
|
|
"firstTokenIds": [
|
|
207,
|
|
17,
|
|
15,
|
|
4,
|
|
280,
|
|
207,
|
|
20,
|
|
15
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 150,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5406ccd886aa",
|
|
"firstTokenIds": [
|
|
207,
|
|
17,
|
|
15,
|
|
4,
|
|
280,
|
|
207,
|
|
20,
|
|
15
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 179,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c6a74b86ecae",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -7,
|
|
"commonPrefixTokens": 112,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 22,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 157,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 29,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1173",
|
|
"index": 13,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 163,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "58d6f7c3c141",
|
|
"firstTokenIds": [
|
|
429,
|
|
4252,
|
|
4085,
|
|
280,
|
|
54078,
|
|
57139,
|
|
7025,
|
|
363
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 248,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "3465cd120aa6",
|
|
"firstTokenIds": [
|
|
429,
|
|
4252,
|
|
4085,
|
|
280,
|
|
54078,
|
|
57139,
|
|
7025,
|
|
363
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 221,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "089b76070587",
|
|
"firstTokenIds": [
|
|
429,
|
|
4252,
|
|
4085,
|
|
280,
|
|
54078,
|
|
57139,
|
|
7025,
|
|
363
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 199,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f6e0428ea361",
|
|
"firstTokenIds": [
|
|
429,
|
|
4252,
|
|
4085,
|
|
280,
|
|
54078,
|
|
57139,
|
|
7025,
|
|
363
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 58,
|
|
"commonPrefixTokens": 17,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -49,
|
|
"commonPrefixTokens": 10,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 85,
|
|
"commonPrefixTokens": 41,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -22,
|
|
"commonPrefixTokens": 10,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0452",
|
|
"index": 14,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 254,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a0ca2283faac",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
254,
|
|
5137,
|
|
4583,
|
|
11,
|
|
395,
|
|
933
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 254,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "be5b1aa682cd",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
254,
|
|
5137,
|
|
4583,
|
|
11,
|
|
395,
|
|
933
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 254,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a0ca2283faac",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
254,
|
|
5137,
|
|
4583,
|
|
11,
|
|
395,
|
|
933
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 246,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "41b1e30b7303",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
254,
|
|
5137,
|
|
4583,
|
|
11,
|
|
395,
|
|
933
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 254,
|
|
"exactTrajectory": true
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -8,
|
|
"commonPrefixTokens": 152,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 213,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -8,
|
|
"commonPrefixTokens": 152,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0357",
|
|
"index": 15,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 299,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8af193f7a509",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
11462,
|
|
254,
|
|
3223
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 399,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d38f3ac8571b",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
11462,
|
|
254,
|
|
3223
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 309,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a704c7519b3d",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
11462,
|
|
254,
|
|
3223
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 323,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "62a9d6f1afb1",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
11462,
|
|
254,
|
|
3223
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 10,
|
|
"commonPrefixTokens": 44,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -76,
|
|
"commonPrefixTokens": 14,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 100,
|
|
"commonPrefixTokens": 50,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 14,
|
|
"commonPrefixTokens": 14,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1050",
|
|
"index": 16,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 88,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "fbf4500cf75d",
|
|
"firstTokenIds": [
|
|
11967,
|
|
578,
|
|
17354,
|
|
207,
|
|
18,
|
|
17,
|
|
20,
|
|
49785
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 122,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ecbd88f6d536",
|
|
"firstTokenIds": [
|
|
11967,
|
|
578,
|
|
17354,
|
|
207,
|
|
18,
|
|
17,
|
|
20,
|
|
49785
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 88,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "fbf4500cf75d",
|
|
"firstTokenIds": [
|
|
11967,
|
|
578,
|
|
17354,
|
|
207,
|
|
18,
|
|
17,
|
|
20,
|
|
49785
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 125,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "004624477d47",
|
|
"firstTokenIds": [
|
|
11967,
|
|
578,
|
|
643,
|
|
2571,
|
|
11516,
|
|
207,
|
|
18,
|
|
17
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 88,
|
|
"exactTrajectory": true
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 3,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 34,
|
|
"commonPrefixTokens": 52,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 37,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0951",
|
|
"index": 17,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 54,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "15362625ba4b",
|
|
"firstTokenIds": [
|
|
1273,
|
|
745,
|
|
418,
|
|
207,
|
|
16,
|
|
15,
|
|
15,
|
|
6769
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 67,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8fc35b0abdd6",
|
|
"firstTokenIds": [
|
|
1273,
|
|
745,
|
|
418,
|
|
207,
|
|
16,
|
|
15,
|
|
15,
|
|
6769
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 107,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "256c27659717",
|
|
"firstTokenIds": [
|
|
1273,
|
|
745,
|
|
418,
|
|
207,
|
|
16,
|
|
15,
|
|
15,
|
|
6769
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 232,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8c88a0c51e11",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
254,
|
|
1604,
|
|
280,
|
|
6769,
|
|
279,
|
|
254
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 53,
|
|
"commonPrefixTokens": 25,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 165,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 13,
|
|
"commonPrefixTokens": 41,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 125,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0906",
|
|
"index": 18,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 136,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "345cd606d60c",
|
|
"firstTokenIds": [
|
|
9663,
|
|
643,
|
|
207,
|
|
22,
|
|
5501,
|
|
24948,
|
|
14253,
|
|
285
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 191,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d9d7eb803d59",
|
|
"firstTokenIds": [
|
|
9663,
|
|
643,
|
|
207,
|
|
22,
|
|
5501,
|
|
24948,
|
|
14253,
|
|
285
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 127,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "3277de277146",
|
|
"firstTokenIds": [
|
|
9663,
|
|
643,
|
|
245,
|
|
3223,
|
|
280,
|
|
207,
|
|
22,
|
|
919
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 223,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "7755880ab652",
|
|
"firstTokenIds": [
|
|
2158,
|
|
8708,
|
|
437,
|
|
2066,
|
|
11,
|
|
395,
|
|
1022,
|
|
933
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -9,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 1,
|
|
"tokenDelta": 32,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 55,
|
|
"commonPrefixTokens": 41,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 96,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0092",
|
|
"index": 19,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 153,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "381e09cb9a85",
|
|
"firstTokenIds": [
|
|
1273,
|
|
7518,
|
|
317,
|
|
207,
|
|
16,
|
|
15,
|
|
285,
|
|
317
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 98,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "8dea3bbc7ac3",
|
|
"firstTokenIds": [
|
|
1273,
|
|
7518,
|
|
317,
|
|
207,
|
|
16,
|
|
15,
|
|
285,
|
|
317
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 82,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5fa78969ef22",
|
|
"firstTokenIds": [
|
|
1273,
|
|
7518,
|
|
317,
|
|
207,
|
|
16,
|
|
15,
|
|
285,
|
|
317
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 131,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d1f7d1f3de0d",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
8708,
|
|
437,
|
|
3458,
|
|
457,
|
|
3458
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -71,
|
|
"commonPrefixTokens": 42,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 33,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -55,
|
|
"commonPrefixTokens": 63,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 49,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0372",
|
|
"index": 20,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 136,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ae3f631d9905",
|
|
"firstTokenIds": [
|
|
9243,
|
|
522,
|
|
661,
|
|
12116,
|
|
5561,
|
|
327,
|
|
207,
|
|
16
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 136,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b741538c8f6a",
|
|
"firstTokenIds": [
|
|
9243,
|
|
522,
|
|
661,
|
|
12116,
|
|
5561,
|
|
327,
|
|
207,
|
|
16
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 136,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "b741538c8f6a",
|
|
"firstTokenIds": [
|
|
9243,
|
|
522,
|
|
661,
|
|
12116,
|
|
5561,
|
|
327,
|
|
207,
|
|
16
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 144,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "206c3c33c1a7",
|
|
"firstTokenIds": [
|
|
9243,
|
|
522,
|
|
661,
|
|
12116,
|
|
5561,
|
|
327,
|
|
207,
|
|
16
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 63,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 8,
|
|
"commonPrefixTokens": 47,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 63,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 8,
|
|
"commonPrefixTokens": 47,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0361",
|
|
"index": 21,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 256,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cee4bc203483",
|
|
"firstTokenIds": [
|
|
429,
|
|
6822,
|
|
29099,
|
|
207,
|
|
22,
|
|
20,
|
|
6593,
|
|
473
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 257,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "adcdcd6996b9",
|
|
"firstTokenIds": [
|
|
429,
|
|
6822,
|
|
29099,
|
|
207,
|
|
22,
|
|
20,
|
|
6593,
|
|
473
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 196,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "1939e08c4ded",
|
|
"firstTokenIds": [
|
|
429,
|
|
6822,
|
|
29099,
|
|
207,
|
|
22,
|
|
20,
|
|
6593,
|
|
473
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 242,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "190248c44054",
|
|
"firstTokenIds": [
|
|
429,
|
|
6822,
|
|
29099,
|
|
207,
|
|
22,
|
|
20,
|
|
6593,
|
|
473
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -60,
|
|
"commonPrefixTokens": 52,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -15,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 1,
|
|
"commonPrefixTokens": 15,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 46,
|
|
"commonPrefixTokens": 52,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0733",
|
|
"index": 22,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 83,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f65911fec2a3",
|
|
"firstTokenIds": [
|
|
17157,
|
|
1503,
|
|
363,
|
|
17,
|
|
15,
|
|
285,
|
|
362,
|
|
7572
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 83,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f65911fec2a3",
|
|
"firstTokenIds": [
|
|
17157,
|
|
1503,
|
|
363,
|
|
17,
|
|
15,
|
|
285,
|
|
362,
|
|
7572
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 83,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f65911fec2a3",
|
|
"firstTokenIds": [
|
|
17157,
|
|
1503,
|
|
363,
|
|
17,
|
|
15,
|
|
285,
|
|
362,
|
|
7572
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 81,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "41adcba6b4f3",
|
|
"firstTokenIds": [
|
|
2158,
|
|
1275,
|
|
636,
|
|
946,
|
|
1266,
|
|
3054,
|
|
1319,
|
|
60282
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 83,
|
|
"exactTrajectory": true
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -2,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 83,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -2,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0687",
|
|
"index": 23,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 282,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "0c81b2f9ed65",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
4500,
|
|
280,
|
|
634
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 217,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a179ca00e6cd",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
4500,
|
|
280,
|
|
634
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 234,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "30bdc63f93ea",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
4500,
|
|
280,
|
|
634
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 202,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "3f9b54299aee",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
13205,
|
|
280,
|
|
254
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -48,
|
|
"commonPrefixTokens": 8,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -15,
|
|
"commonPrefixTokens": 5,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -65,
|
|
"commonPrefixTokens": 8,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -32,
|
|
"commonPrefixTokens": 5,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0584",
|
|
"index": 24,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 298,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "9dce4686d271",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
2973,
|
|
1224,
|
|
254,
|
|
2066,
|
|
3458
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 314,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "93961697cdf4",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
2973,
|
|
1224,
|
|
254,
|
|
2066,
|
|
3458
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 290,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "d8bf77d7440c",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 284,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "da9fed9acdd7",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
395,
|
|
933,
|
|
276,
|
|
6983,
|
|
946,
|
|
1313
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 1,
|
|
"tokenDelta": -8,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -30,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 1,
|
|
"tokenDelta": 16,
|
|
"commonPrefixTokens": 10,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": -1,
|
|
"tokenDelta": -6,
|
|
"commonPrefixTokens": 2,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0",
|
|
"T1",
|
|
"T2",
|
|
"T3"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1152",
|
|
"index": 25,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 149,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "afc53c199efe",
|
|
"firstTokenIds": [
|
|
429,
|
|
1022,
|
|
48441,
|
|
15006,
|
|
643,
|
|
245,
|
|
207,
|
|
17
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 244,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "cc57dc1d59c9",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
7599,
|
|
280,
|
|
254
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 176,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "f80e45684bc5",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
7599,
|
|
280,
|
|
254
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 196,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2600b7dd1901",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
2973,
|
|
1224,
|
|
254,
|
|
2066,
|
|
25
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 27,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -48,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 95,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 20,
|
|
"commonPrefixTokens": 3,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1021",
|
|
"index": 26,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 190,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6447b9536055",
|
|
"firstTokenIds": [
|
|
207,
|
|
19,
|
|
15,
|
|
4,
|
|
280,
|
|
254,
|
|
4313,
|
|
418
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 180,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "98267904a531",
|
|
"firstTokenIds": [
|
|
207,
|
|
19,
|
|
15,
|
|
4,
|
|
280,
|
|
254,
|
|
4313,
|
|
418
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 152,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "ead34de5c2cf",
|
|
"firstTokenIds": [
|
|
338,
|
|
2228,
|
|
317,
|
|
2049,
|
|
276,
|
|
752,
|
|
207,
|
|
22
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 297,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "a3792c95d26a",
|
|
"firstTokenIds": [
|
|
5904,
|
|
11,
|
|
1632,
|
|
6,
|
|
82,
|
|
1275,
|
|
636,
|
|
946
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -38,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 117,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -10,
|
|
"commonPrefixTokens": 71,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 145,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1194",
|
|
"index": 27,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 97,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c9bbda61f523",
|
|
"firstTokenIds": [
|
|
1273,
|
|
245,
|
|
51885,
|
|
643,
|
|
1853,
|
|
2602,
|
|
372,
|
|
1266
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 97,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c9bbda61f523",
|
|
"firstTokenIds": [
|
|
1273,
|
|
245,
|
|
51885,
|
|
643,
|
|
1853,
|
|
2602,
|
|
372,
|
|
1266
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 78,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "838cb016b96a",
|
|
"firstTokenIds": [
|
|
1273,
|
|
245,
|
|
51885,
|
|
643,
|
|
1853,
|
|
2602,
|
|
372,
|
|
1266
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 209,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "2d8a1d1b7c5a",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
2973,
|
|
1224,
|
|
254,
|
|
2066,
|
|
881
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -19,
|
|
"commonPrefixTokens": 68,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 112,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 97,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 131,
|
|
"commonPrefixTokens": 0,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0399",
|
|
"index": 28,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 85,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "736f1945483a",
|
|
"firstTokenIds": [
|
|
1273,
|
|
82272,
|
|
317,
|
|
207,
|
|
18,
|
|
15,
|
|
1555,
|
|
1712
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 86,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "fdd7523b771b",
|
|
"firstTokenIds": [
|
|
1273,
|
|
82272,
|
|
317,
|
|
207,
|
|
18,
|
|
15,
|
|
1555,
|
|
1712
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 86,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "fdd7523b771b",
|
|
"firstTokenIds": [
|
|
1273,
|
|
82272,
|
|
317,
|
|
207,
|
|
18,
|
|
15,
|
|
1555,
|
|
1712
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 86,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "fdd7523b771b",
|
|
"firstTokenIds": [
|
|
1273,
|
|
82272,
|
|
317,
|
|
207,
|
|
18,
|
|
15,
|
|
1555,
|
|
1712
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 1,
|
|
"commonPrefixTokens": 28,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 86,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 1,
|
|
"commonPrefixTokens": 28,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 86,
|
|
"exactTrajectory": true
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0214",
|
|
"index": 29,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 512,
|
|
"naturalEos": false,
|
|
"truncated": true,
|
|
"trajectoryHash": "6820a91df8d1",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
3744,
|
|
280,
|
|
2354
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 494,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "c61d867bb525",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
3744,
|
|
280,
|
|
2354
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 308,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "318257aed6de",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
3744,
|
|
280,
|
|
2354
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 471,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "60e00d082884",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
9211,
|
|
254,
|
|
3744,
|
|
280,
|
|
2354
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -204,
|
|
"commonPrefixTokens": 94,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -23,
|
|
"commonPrefixTokens": 65,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -18,
|
|
"commonPrefixTokens": 12,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 163,
|
|
"commonPrefixTokens": 12,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/1084",
|
|
"index": 30,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 174,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e19fa9207aa3",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
6049,
|
|
7421,
|
|
5698,
|
|
1376,
|
|
5144
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 174,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e19fa9207aa3",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
6049,
|
|
7421,
|
|
5698,
|
|
1376,
|
|
5144
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 174,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "e19fa9207aa3",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
6049,
|
|
7421,
|
|
5698,
|
|
1376,
|
|
5144
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 188,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "540b2e41129b",
|
|
"firstTokenIds": [
|
|
3960,
|
|
6,
|
|
82,
|
|
6049,
|
|
7421,
|
|
5698,
|
|
1376,
|
|
5144
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 174,
|
|
"exactTrajectory": true
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 14,
|
|
"commonPrefixTokens": 71,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 174,
|
|
"exactTrajectory": true
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 14,
|
|
"commonPrefixTokens": 71,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
},
|
|
{
|
|
"id": "gsm8k/test/0964",
|
|
"index": 31,
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 68,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "323a943a78a8",
|
|
"firstTokenIds": [
|
|
1273,
|
|
1319,
|
|
15341,
|
|
317,
|
|
207,
|
|
17,
|
|
15,
|
|
4063
|
|
]
|
|
},
|
|
"s1_eos": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 98,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "5b09328f0ddd",
|
|
"firstTokenIds": [
|
|
1273,
|
|
1319,
|
|
15341,
|
|
317,
|
|
207,
|
|
17,
|
|
15,
|
|
4063
|
|
]
|
|
},
|
|
"s0_period": {
|
|
"success": false,
|
|
"outcome": "wrong_numeric_answer",
|
|
"tokens": 68,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "24e0750594b9",
|
|
"firstTokenIds": [
|
|
1273,
|
|
1319,
|
|
15341,
|
|
317,
|
|
207,
|
|
17,
|
|
15,
|
|
4063
|
|
]
|
|
},
|
|
"s1_period": {
|
|
"success": true,
|
|
"outcome": "passed",
|
|
"tokens": 78,
|
|
"naturalEos": true,
|
|
"truncated": false,
|
|
"trajectoryHash": "6ad7b8c26f21",
|
|
"firstTokenIds": [
|
|
1273,
|
|
1319,
|
|
15341,
|
|
317,
|
|
207,
|
|
17,
|
|
15,
|
|
4063
|
|
]
|
|
}
|
|
},
|
|
"contrasts": {
|
|
"period_at_s0": {
|
|
"successDelta": -1,
|
|
"tokenDelta": 0,
|
|
"commonPrefixTokens": 16,
|
|
"exactTrajectory": false
|
|
},
|
|
"period_at_s1": {
|
|
"successDelta": 0,
|
|
"tokenDelta": -20,
|
|
"commonPrefixTokens": 9,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_eos": {
|
|
"successDelta": 0,
|
|
"tokenDelta": 30,
|
|
"commonPrefixTokens": 17,
|
|
"exactTrajectory": false
|
|
},
|
|
"system_at_period": {
|
|
"successDelta": 1,
|
|
"tokenDelta": 10,
|
|
"commonPrefixTokens": 9,
|
|
"exactTrajectory": false
|
|
}
|
|
},
|
|
"diagnosticTapes": [
|
|
"T0"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"outcomes": {
|
|
"code": {
|
|
"assertion_failed": 47,
|
|
"extract_failed": 2,
|
|
"passed": 59,
|
|
"runtime_error": 16,
|
|
"syntax_error": 1,
|
|
"timeout": 3
|
|
},
|
|
"math": {
|
|
"passed": 71,
|
|
"wrong_numeric_answer": 57
|
|
}
|
|
},
|
|
"reproduction": {
|
|
"cells": 64,
|
|
"all_preregistered_fields_exact": 64,
|
|
"by_field": {
|
|
"run_seed_exact": 64,
|
|
"prompt_hash_exact": 64,
|
|
"uniform_run_hash_exact": 64,
|
|
"uniform_output_prefix_hash_exact": 64,
|
|
"uniform_steps_exact": 64,
|
|
"generated_token_ids_exact": 64,
|
|
"decoded_text_exact": 64,
|
|
"eos_state_exact": 64,
|
|
"truncation_state_exact": 64,
|
|
"cpu_rng_pre_state_exact": 64,
|
|
"cuda_rng_pre_state_exact": 64,
|
|
"torch_rng_unchanged_exact": 64
|
|
}
|
|
},
|
|
"bootstrap": {
|
|
"resamples": 10000,
|
|
"seed": 1364512825,
|
|
"rng": "numpy.random.default_rng reset per domain",
|
|
"unit": "selected task",
|
|
"paired_conditions": true,
|
|
"tape": "T0",
|
|
"interval_label": "selected-task resampling band for the fixed 32-task frame and T0",
|
|
"not": [
|
|
"benchmark-population confidence interval",
|
|
"model-ability confidence interval",
|
|
"generation-seed uncertainty interval",
|
|
"causal-effect confidence interval"
|
|
]
|
|
},
|
|
"uniformExample": {
|
|
"sourceId": "HumanEval/31",
|
|
"tape": "T0",
|
|
"uniformUint64FirstEightHex": [
|
|
"72223be6d80669ef",
|
|
"41a829ee24b9f837",
|
|
"1fc9148ea2f2ac2c",
|
|
"f6349bcb6cec7b5b",
|
|
"bd0b252f46fec99d",
|
|
"ae62463483fd351b",
|
|
"836e54d1468e348e",
|
|
"e47b1da3da2685d8"
|
|
],
|
|
"uniformFloat32FirstEight": [
|
|
0.4458348751068115,
|
|
0.25647222995758057,
|
|
0.1241619884967804,
|
|
0.961740255355835,
|
|
0.7384513020515442,
|
|
0.6811870336532593,
|
|
0.5134022831916809,
|
|
0.8925036191940308
|
|
],
|
|
"conditions": {
|
|
"s0_eos": {
|
|
"generatedTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
262,
|
|
62,
|
|
6620,
|
|
63
|
|
],
|
|
"generatedTokens": 394,
|
|
"naturalEos": true
|
|
},
|
|
"s1_eos": {
|
|
"generatedTokenIds": [
|
|
2158,
|
|
6983,
|
|
565,
|
|
245,
|
|
1604,
|
|
317,
|
|
9966,
|
|
11
|
|
],
|
|
"generatedTokens": 344,
|
|
"naturalEos": true
|
|
},
|
|
"s0_period": {
|
|
"generatedTokenIds": [
|
|
2158,
|
|
4353,
|
|
254,
|
|
2030,
|
|
262,
|
|
62,
|
|
6620,
|
|
63
|
|
],
|
|
"generatedTokens": 420,
|
|
"naturalEos": true
|
|
},
|
|
"s1_period": {
|
|
"generatedTokenIds": [
|
|
977,
|
|
317,
|
|
62,
|
|
6620,
|
|
7,
|
|
77,
|
|
1780,
|
|
185
|
|
],
|
|
"generatedTokens": 59,
|
|
"naturalEos": true
|
|
}
|
|
}
|
|
},
|
|
"deviations": [
|
|
{
|
|
"id": "prompt-hash-correction",
|
|
"severity": "corrected-before-output",
|
|
"summary": "The first manifest mislabeled routing-probe hashes as Chat prompt hashes; all 256 Chat hashes were corrected before model output."
|
|
},
|
|
{
|
|
"id": "gold-loaded-in-runner",
|
|
"severity": "reported-process-deviation",
|
|
"summary": "The reused runner loaded gold for a post-decode narrow task_score before generation finished. Gold never entered prompts, logits, sampling, selection, or the authoritative evaluator."
|
|
}
|
|
],
|
|
"evidenceBoundary": [
|
|
"HumanEval and GSM8K remain separate.",
|
|
"Selected-task bands cover only the frozen 32-task frame under T0.",
|
|
"T1-T3 are sensitivity diagnostics, not extra independent tasks.",
|
|
"The explicit sampler uses the official .3/.95 distribution but is not a torch.multinomial trajectory.",
|
|
"Period prompts are counterfactual and not official-valid chats."
|
|
]
|
|
}
|