{ "architecture": "block", "artifacts": { "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" }, "batch_size": 32, "canonical_sha256_without_self": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78", "depth": 32, "diagnostics": [ { "forward_identity_gate": { "comparisons": { "detached_learned": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_value_backward": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "initialization_negative_control": { "comparisons": { "detached_learned": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "uniform_value_backward": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } } }, "passed": true }, "loss_scale_gate": { "checks": { "attention_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "mlp_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_attention_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_mlp_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_attention_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_mlp_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } } }, "passed": true }, "modes": { "detached_learned": { "bits_per_byte": 8.038491115106986, "forward": { "activation_sha256": { "attention_branch_output": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", "mlp_branch_output": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", "post_attention_state": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", "post_mlp_state": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", "pre_attention_input": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", "pre_mlp_input": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd" }, "logits_sha256": "5e5a50bae6f74c41645e35a857f0cba662c8b3f5ba9a8a4635e860fdaa3b0c88", "loss_nats": 5.571857452392578, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.571857452392578, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0023513585329055786, 0.0023208872880786657, 0.002778682392090559, 0.002497476991266012, 0.002628475893288851, 0.002920303260907531, 0.002932395786046982, 0.0032228652853518724, 0.003561305580660701, 0.003354643704369664, 0.0034162327647209167, 0.0036349669098854065, 0.0035210708156228065, 0.0038089773152023554, 0.003915763925760984, 0.0037244565319269896, 0.004162859171628952, 0.004167158622294664, 0.004440344404429197, 0.0041700867004692554, 0.004737259820103645, 0.004552462603896856, 0.004706811159849167, 0.005186605267226696, 0.005580419674515724, 0.004905648995190859, 0.005227713380008936, 0.005349088925868273, 0.005235649179667234, 0.004925078712403774, 0.005810525268316269, 0.005564783234149218 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024914435198297724, "first_to_last_ratio": 5.005386356076312, "imbalance_abs_log_ratio": 1.61051460380912, "last_quartile_mean": 4.977524895366514e-05, "mean": 0.00012233095264946314, "normalized": [ 2.946204230178919, 2.5883907308658176, 2.285503889899197, 2.0157125547168477, 1.8168856694000293, 1.687577213316311, 1.555459627812154, 1.3974018168018878, 1.2756749767931916, 1.1684180974434406, 1.0925035606597346, 1.0000212864934148, 0.922022236502042, 0.863306019148705, 0.7954975128821132, 0.747136294956448, 0.6955650087701328, 0.6567702950630189, 0.6067855157189815, 0.5734103460233209, 0.549704279492861, 0.527795032775898, 0.5009080420334019, 0.4762252632696638, 0.45754883501456406, 0.43797117166433, 0.4282553488048722, 0.41570642703193256, 0.40058854659018955, 0.39033614610916745, 0.3703926654711839, 0.3543213582962283 ], "peak_layer": 1, "peak_normalized": 2.946204230178919, "population_cv": 0.6883820030702144, "rest_mean": 0.00013360270127628205, "spike_contrast": 0.46004691130861425, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.146351006464101e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036041197017766535, 0.0003166403039358556, 0.00027958786813542247, 0.0002465840370859951, 0.00022226135479286313, 0.0002064429281745106, 0.00019028085807804018, 0.0001709454954834655, 0.00015605453518219292, 0.00014293369895312935, 0.00013364700134843588, 0.00012233355664648116, 0.0001127918585552834, 0.00010560904775047675, 9.731396858114749e-05, 9.139789472101256e-05, 8.508913015248254e-05, 8.03433358669281e-05, 7.42286501917988e-05, 7.014583388809115e-05, 6.724584818584844e-05, 6.456566916313022e-05, 6.127655797172338e-05, 5.825709013151936e-05, 5.597238487098366e-05, 5.3577430662699044e-05, 5.2388884796528146e-05, 5.085376324132085e-05, 4.900437852484174e-05, 4.775019260705449e-05, 4.5310487621463835e-05, 4.334446930442937e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002986813742609229, "first_to_last_ratio": 2.3520308544062645, "imbalance_abs_log_ratio": 0.8552791483203458, "last_quartile_mean": 0.0001269887143280357, "mean": 0.00018719729814620223, "normalized": [ 2.1746021240914017, 1.94336263487061, 1.7490368812517914, 1.5797937988337565, 1.4558020735282082, 1.372777068369229, 1.290232936323773, 1.1987380034727069, 1.1282687150208124, 1.0677045109732362, 1.0238712087163733, 0.9743741602296072, 0.9332108464677616, 0.9012687760530631, 0.8680188022864366, 0.8423083374767277, 0.8168968442148058, 0.7977443195284458, 0.774889242886524, 0.759928616461841, 0.7477907292059667, 0.7367409087088999, 0.7235787757136496, 0.7121132246295925, 0.7033660136533255, 0.6946945949961153, 0.6884818791691059, 0.6826464924874314, 0.674695217464828, 0.6690870522688999, 0.6604539573176332, 0.6535212533274407 ], "peak_layer": 1, "peak_normalized": 2.1746021240914017, "population_cv": 0.3960989175249597, "rest_mean": 0.00019674027215741161, "spike_contrast": 0.68956516628749, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013566523848567157, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004070796421729028, 0.0003637922345660627, 0.0003274149785283953, 0.0002957331307698041, 0.0002725222148001194, 0.000256980158155784, 0.0002415281196590513, 0.00022440051543526351, 0.00021120885503478348, 0.00019987139967270195, 0.00019166592392139137, 0.0001824002101784572, 0.00017469454905949533, 0.00016871507978066802, 0.00016249077452812344, 0.00015767784498166293, 0.00015292088210117072, 0.0001493355812272057, 0.00014505717263091356, 0.0001422565837856382, 0.00013998440408613533, 0.0001379159075440839, 0.00013545199180953205, 0.0001333056716248393, 0.00013166821736376733, 0.000130044951220043, 0.0001288819476030767, 0.00012778957898262888, 0.00012630112178158015, 0.00012525128840934485, 0.00012363519635982811, 0.00012233741290401667 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.264873631298542, "first_to_last_ratio": 2.3520309299879743, "imbalance_abs_log_ratio": 0.8552791804550044, "last_quartile_mean": 0.11261485889554024, "mean": 0.1660084358882159, "normalized": [ 2.1746021175238424, 1.943362568695452, 1.7490370652499811, 1.5797937888156908, 1.4558020518807187, 1.3727770906872316, 1.2902329816871188, 1.1987379486864336, 1.128268732118104, 1.067704514212167, 1.0238712059346726, 0.9743741284397964, 0.9332107750795899, 0.9012687970282692, 0.8680188148399602, 0.8423083473020997, 0.8168968752199287, 0.7977442908994873, 0.7748892254753884, 0.7599285915530729, 0.7477907763310051, 0.7367409150028685, 0.723578786768912, 0.7121132849551643, 0.7033659848486875, 0.694694622945607, 0.6884818276985868, 0.6826464794318051, 0.6746951840389442, 0.6690870220520635, 0.6604539434158253, 0.6535212611815249 ], "peak_layer": 1, "peak_normalized": 2.1746021175238424, "population_cv": 0.3960989241805914, "rest_mean": 0.17447124007675383, "spike_contrast": 0.6895651868880185, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12030929327011108, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.3610022962093353, 0.3226145803928375, 0.2903549075126648, 0.2622590959072113, 0.24167542159557343, 0.22789257764816284, 0.2141895592212677, 0.19900061190128326, 0.1873021274805069, 0.17724795639514923, 0.1699712574481964, 0.1617543250322342, 0.15492086112499237, 0.1496182233095169, 0.1440984457731247, 0.13983029127120972, 0.13561177253723145, 0.13243228197097778, 0.1286381483078003, 0.1261545568704605, 0.12413957715034485, 0.12230520695447922, 0.12012018263339996, 0.11821681261062622, 0.11676468700170517, 0.11532516777515411, 0.11429379135370255, 0.1133250743150711, 0.1120050922036171, 0.11107409000396729, 0.1096409261226654, 0.10849004238843918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010120303977600997, "first_to_last_ratio": 2.7021675066537045, "imbalance_abs_log_ratio": 0.994054231196686, "last_quartile_mean": 3.7452541164384456e-05, "mean": 6.280763545873924e-05, "normalized": [ 1.9944816746100333, 1.8472480968114133, 1.7373223217437128, 1.6248488809553971, 1.5471371990089653, 1.475823179261819, 1.3807460832170038, 1.2829322622274897, 1.184046410564159, 1.1043973349513228, 1.0901910170788929, 1.0456150919711364, 0.9888694060739084, 0.9664075020831666, 0.9198699716804313, 0.9041232034205934, 0.8692950074065211, 0.8446819175937094, 0.7923092109368705, 0.7644112750281646, 0.7503465778061283, 0.730017034806627, 0.7090250315296005, 0.6754099655367306, 0.6572034579968898, 0.6267819521510305, 0.6185439744898635, 0.6068927943478243, 0.594580659914726, 0.5858446061910916, 0.5555293303136664, 0.5250675682911103 ], "peak_layer": 1, "peak_normalized": 1.9944816746100333, "population_cv": 0.40483316550850257, "rest_mean": 6.624576713807915e-05, "spike_contrast": 0.6678422833882899, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4241724390303715e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012526867794804275, 0.0001160212850663811, 0.0001091171070584096, 0.00010205291619058698, 9.717202920001e-05, 9.269296424463391e-05, 8.67213966557756e-05, 8.057794184423983e-05, 7.43671553209424e-05, 6.936458521522582e-05, 6.847231998108327e-05, 6.567261152667925e-05, 6.210854917299002e-05, 6.0697770095430315e-05, 5.777485785074532e-05, 5.6785840570228174e-05, 5.4598363931290805e-05, 5.305247395881452e-05, 4.9763068091124296e-05, 4.801086470251903e-05, 4.712749432655983e-05, 4.585064380080439e-05, 4.453218571143225e-05, 4.242090290063061e-05, 4.1277395212091506e-05, 3.936669236281887e-05, 3.8849284464959055e-05, 3.811750138993375e-05, 3.734420533874072e-05, 3.679551446111873e-05, 3.4891483664978296e-05, 3.297825242043473e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021292950441420544, "first_to_last_ratio": 1.6819067487661699, "imbalance_abs_log_ratio": 0.5199281193278776, "last_quartile_mean": 0.0001266000654140953, "mean": 0.0001612693722563563, "normalized": [ 1.5193752902631465, 1.4431842082185322, 1.3854135927323608, 1.3303138916593422, 1.2819858609596173, 1.2449680699964556, 1.2025089225516594, 1.1549255897117912, 1.1161361523436344, 1.0820631347545688, 1.0617857510438193, 1.0341003696475617, 1.0077971864601107, 0.9864740856164548, 0.9634556759892069, 0.9447446572547388, 0.9240188125766456, 0.9078710595764491, 0.88634376010722, 0.8724890232241534, 0.8607832896405347, 0.849628883322604, 0.8359853832855236, 0.8234683594489096, 0.8139294954165563, 0.8040629029670215, 0.7969416676465471, 0.7902390259734583, 0.7809695092222964, 0.7743360767166025, 0.7639958481267083, 0.7557044635457686 ], "peak_layer": 1, "peak_normalized": 1.5193752902631465, "population_cv": 0.21463654613558777, "rest_mean": 0.00016614451305940747, "spike_contrast": 0.8122062500591205, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013494361191987992, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024502869928255677, 0.00023274141130968928, 0.00022342478041537106, 0.0002145388862118125, 0.00020674505503848195, 0.00020077521912753582, 0.00019392785907257348, 0.00018625412485562265, 0.00017999857664108276, 0.00017450364248361439, 0.00017123352154158056, 0.00016676871746312827, 0.0001625268196221441, 0.00015908805653452873, 0.0001553758920636028, 0.0001523583778180182, 0.0001490159338572994, 0.000146411795867607, 0.00014294010179582983, 0.0001407057570759207, 0.00013881798076909035, 0.00013701911666430533, 0.0001348188379779458, 0.0001328002254012972, 0.00013126189878676087, 0.0001296707196161151, 0.00012852228246629238, 0.00012744135165121406, 0.0001259464625036344, 0.00012487669300753623, 0.00012320913083385676, 0.00012187198444735259 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016320452232321259, "first_to_last_ratio": 1.2889611434881496, "imbalance_abs_log_ratio": 0.25383657880769345, "last_quartile_mean": 0.0001266171002498595, "mean": 0.0001437355554116948, "normalized": [ 1.188749445631, 1.1713423911172207, 1.1549430883999143, 1.1417765084687417, 1.1273283201397843, 1.1145504039766105, 1.0995991496765185, 1.0853099095412984, 1.0687879028680622, 1.0554743211679045, 1.0473286818228558, 1.0363809984280803, 1.0249947395076546, 1.015899258153915, 1.005968438235508, 0.9970397006238613, 0.9860146691908593, 0.977410712347053, 0.964005406402611, 0.9549550784847994, 0.9474810711889571, 0.940343993567483, 0.9294782140508817, 0.9176125795270506, 0.909513915953909, 0.8990879268536105, 0.8931112722873075, 0.8859642730588357, 0.8770245001904262, 0.8704731011044752, 0.8603223859779169, 0.851727642054894 ], "peak_layer": 1, "peak_normalized": 1.188749445631, "population_cv": 0.0978236443065259, "rest_mean": 0.0001456284474072702, "spike_contrast": 0.9168122095142401, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001335139386355877, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017086556181311607, 0.00016836354916449636, 0.00016600638628005981, 0.00016411388060078025, 0.0001620371622266248, 0.00016020052134990692, 0.0001580514945089817, 0.00015599762264173478, 0.00015362282283604145, 0.0001517091877758503, 0.00015053836978040636, 0.0001489647984271869, 0.00014732818817719817, 0.0001460208441130817, 0.00014459343219641596, 0.0001433100551366806, 0.00014172536612022668, 0.00014048867160454392, 0.00013856185250915587, 0.00013726099859923124, 0.0001361867180094123, 0.00013516086619347334, 0.00013359906733967364, 0.0001318935537710786, 0.0001307294878643006, 0.00012923090253025293, 0.00012837184476666152, 0.0001273445668630302, 0.00012605960364453495, 0.0001251179346581921, 0.00012365891598165035, 0.00012242354569025338 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0004412600719660986, "first_to_last_ratio": 3.098863205201153, "imbalance_abs_log_ratio": 1.131035336237652, "last_quartile_mean": 0.00014239417578210123, "mean": 0.0002548141455918085, "normalized": [ 2.1785577639630773, 1.9956115105027457, 1.9014578504944815, 1.7829105267225334, 1.6475854563109211, 1.565091756282257, 1.45799103716316, 1.3243443696030393, 1.2370613678820481, 1.1609137324491365, 1.1300355744023234, 1.0677196003316773, 1.0048909110245325, 0.948494062464312, 0.8853730946340664, 0.8440976906413151, 0.7906677717627395, 0.747069444517614, 0.6991826523616375, 0.6712814178889162, 0.6509454457824981, 0.6314362826751919, 0.6113540983289655, 0.5954000197507868, 0.5831757760496624, 0.5733601746441271, 0.5653129211221045, 0.559900286080882, 0.5532114024202266, 0.5496528914136888, 0.545014697375157, 0.5408984129541747 ], "peak_layer": 1, "peak_normalized": 2.1785577639630773, "population_cv": 0.4822008017525282, "rest_mean": 0.00027300682214209465, "spike_contrast": 0.573515676244785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015657369222026317, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005551273352466524, 0.0005085100419819355, 0.000484518357552588, 0.00045431082253344357, 0.00041982808033935726, 0.0003988075186498463, 0.0003715167404152453, 0.0003374616790097207, 0.000315220735501498, 0.0002958172408398241, 0.0002879490493796766, 0.0002720700576901436, 0.0002560604189056903, 0.00024168970412574708, 0.00022560558863915503, 0.00021508803183678538, 0.0002014733327087015, 0.0001903638622025028, 0.0001781616301741451, 0.0001710520009510219, 0.00016587010759394616, 0.0001608988968655467, 0.00015578167221974581, 0.00015171634731814265, 0.00014860143710393459, 0.00014610028301831335, 0.0001440497289877385, 0.0001426705130143091, 0.00014096609083935618, 0.0001400593318976462, 0.0001388774544466287, 0.00013782856694888324 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027412420604377985, 0.002683233004063368, 0.002711885841563344, 0.0027249811682850122, 0.0027012599166482687, 0.002700963756069541, 0.0026683746837079525, 0.002738022478297353, 0.0027370047755539417, 0.002654933836311102, 0.002663377905264497, 0.002684312639757991, 0.0027356327045708895, 0.002652956871315837, 0.002679797587916255, 0.002698024269193411, 0.0026484597474336624, 0.0026680538430809975, 0.002654345240443945, 0.002591830212622881, 0.002620960585772991, 0.002651234855875373, 0.0026270023081451654, 0.002615687670186162, 0.00258000404573977, 0.002626793459057808, 0.002573782578110695, 0.002592637902125716, 0.002581534907221794, 0.002551236655563116, 0.002563327783718705, 0.002589041367173195 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024784023116808385, "first_to_last_ratio": 4.996872150813687, "imbalance_abs_log_ratio": 1.6088121468463858, "last_quartile_mean": 4.959907391821616e-05, "mean": 0.00012175017718618619, "normalized": [ 2.94187592163941, 2.5796687009574604, 2.293010133276413, 2.014581184592976, 1.8111575385173901, 1.6842185280489494, 1.5559062012549134, 1.4047478191310576, 1.2749623745204015, 1.1699150302730006, 1.0919472369387608, 1.0017565099771182, 0.923449750141626, 0.8652102016053241, 0.794546741948017, 0.743905553038253, 0.6942834150416333, 0.6562793060663384, 0.6082995898528493, 0.5744438165059796, 0.5512557468449423, 0.5265581646918154, 0.5020472770644135, 0.47690127545745264, 0.4572061982209032, 0.43893553328289897, 0.42961039869683454, 0.41603121061550513, 0.3999104909112459, 0.3911954601706125, 0.3717234123738856, 0.35445927834161867 ], "peak_layer": 1, "peak_normalized": 2.94187592163941, "population_cv": 0.6874421460933116, "rest_mean": 0.0001329603533290706, "spike_contrast": 0.46040210094136547, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.121522601461038e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000358173914719373, 0.0003140751214232296, 0.00027917439001612365, 0.0002452756161801517, 0.00022050875122658908, 0.0002050539042102173, 0.00018943185568787158, 0.0001710282958811149, 0.00015522689500357956, 0.00014243736222852021, 0.00013294476957526058, 0.00012196403258712962, 0.00011243017070228234, 0.0001053394953487441, 9.673620661487803e-05, 9.057063289219514e-05, 8.45291287987493e-05, 7.990212179720402e-05, 7.406058284686878e-05, 6.993863644311205e-05, 6.711548485327512e-05, 6.410854985006154e-05, 6.112434493843466e-05, 5.806281478726305e-05, 5.566493564401753e-05, 5.3440478950506076e-05, 5.23051421623677e-05, 5.0651873607421294e-05, 4.8689173127058893e-05, 4.762811659020372e-05, 4.525739132077433e-05, 4.315547994337976e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002960815872938838, "first_to_last_ratio": 2.3434901925615312, "imbalance_abs_log_ratio": 0.8516413534183918, "last_quartile_mean": 0.00012634214908757713, "mean": 0.00018583460405352525, "normalized": [ 2.1691817718266027, 1.9356770461635995, 1.7512361881653136, 1.5771501673188, 1.4517248962140685, 1.3691574500653627, 1.2898282592750647, 1.2020702391870204, 1.1274711846152252, 1.06796916249687, 1.023464646242978, 0.9751293351351744, 0.9332415616222968, 0.9026335726010739, 0.8668647658197504, 0.8407791679025106, 0.8171800119835181, 0.7975248027052753, 0.7770671945621679, 0.7612080123915693, 0.7492445405290613, 0.7365475788483447, 0.7249228575947372, 0.7138182430319103, 0.7048361054118131, 0.6962681268232633, 0.6911654119184109, 0.6833348384348117, 0.6760742523950837, 0.6702828385523486, 0.6624247014500871, 0.654521068715886 ], "peak_layer": 1, "peak_normalized": 2.1691817718266027, "population_cv": 0.3943796352106793, "rest_mean": 0.00019526833030222742, "spike_contrast": 0.6908057343541223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013489248231053353, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004031090356875211, 0.0003597157774493098, 0.0003254402836319059, 0.0002930890768766403, 0.00026978072128258646, 0.00025443683261983097, 0.00023969472385942936, 0.00022338624694384634, 0.00020952316117472947, 0.0001984656264539808, 0.0001901951472973451, 0.00018121277389582247, 0.0001734285760903731, 0.00016774055256973952, 0.0001610934705240652, 0.0001562458637636155, 0.00015186032396741211, 0.00014820770593360066, 0.00014440597442444414, 0.00014145878958515823, 0.00013923556252848357, 0.0001368760276818648, 0.00013471575221046805, 0.00013265213056001812, 0.00013098293857183307, 0.00012939071166329086, 0.00012844245065934956, 0.00012698725913651288, 0.00012563799100462347, 0.0001245617459062487, 0.00012310143210925162, 0.00012163266364950687 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.26256811060011387, "first_to_last_ratio": 2.3434901455089974, "imbalance_abs_log_ratio": 0.8516413333404181, "last_quartile_mean": 0.11204148270189762, "mean": 0.16479998547583818, "normalized": [ 2.1691817568110046, 1.935676923537455, 1.7512361570514487, 1.5771502517930829, 1.4517248703109182, 1.369157420355369, 1.2898282606548555, 1.2020702692325518, 1.1274712340116186, 1.0679692103147351, 1.0234646460792782, 0.975129361875517, 0.9332415430463463, 0.9026335787242323, 0.8668647305888977, 0.8407791967519535, 0.8171800231493583, 0.7975247749486774, 0.7770672329681078, 0.7612079835803747, 0.7492445111174014, 0.7365475542475244, 0.7249228843083022, 0.7138182179223472, 0.7048361082067401, 0.6962681206044983, 0.6911654218524023, 0.6833348067031066, 0.676074287607254, 0.6702828621875807, 0.6624247142471237, 0.6545210852099367 ], "peak_layer": 1, "peak_normalized": 2.1691817568110046, "population_cv": 0.3943796257753467, "rest_mean": 0.17316590856622766, "spike_contrast": 0.6908057237027376, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11962400078773498, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.35748112201690674, 0.3189995288848877, 0.28860369324684143, 0.2599143385887146, 0.2392442375421524, 0.22563712298870087, 0.2125636786222458, 0.19810116291046143, 0.1858072429895401, 0.17600131034851074, 0.16866695880889893, 0.16070130467414856, 0.1537981927394867, 0.14875400066375732, 0.1428592950105667, 0.13856039941310883, 0.13467125594615936, 0.13143207132816315, 0.12806066870689392, 0.12544706463813782, 0.12347548454999924, 0.12138302624225616, 0.11946728080511093, 0.11763723194599152, 0.11615698039531708, 0.11474497616291046, 0.11390405148267746, 0.11261356621980667, 0.11141703277826309, 0.11046260595321655, 0.10916758328676224, 0.10786506533622742 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.0001010398200378404, "first_to_last_ratio": 2.701478580108855, "imbalance_abs_log_ratio": 0.9937992453824708, "last_quartile_mean": 3.740167358046165e-05, "mean": 6.270599271829269e-05, "normalized": [ 1.9907220064146427, 1.8444741506775382, 1.742884685905627, 1.6239253294262157, 1.5436270879110234, 1.4787029723792133, 1.3841644782908762, 1.2821103017907618, 1.1824374683113494, 1.1042100658007115, 1.0908679078034196, 1.0498354286471192, 0.9920033594422748, 0.9660018389944104, 0.9202838356691797, 0.8989347700163286, 0.8686518679586676, 0.8410583182307008, 0.7938538755561205, 0.7653997098760733, 0.7511903220519177, 0.7265360595007238, 0.7106247592487483, 0.6758121071782397, 0.6556340454607258, 0.6278169583683687, 0.6205082786340235, 0.6073860029650574, 0.5911469097961847, 0.5867782134359579, 0.5577710930141501, 0.5246457912436486 ], "peak_layer": 1, "peak_normalized": 1.9907220064146427, "population_cv": 0.40468494452628273, "rest_mean": 6.614368087159366e-05, "spike_contrast": 0.6673725457789733, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4142476690467446e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001248301996383816, 0.00011565958266146481, 0.0001092893144232221, 0.00010182984988205135, 9.679466893430799e-05, 9.272353781852871e-05, 8.679540769662708e-05, 8.039599924813956e-05, 7.414591527776793e-05, 6.92405883455649e-05, 6.84039550833404e-05, 6.583097274415195e-05, 6.220455543370917e-05, 6.0574104281840846e-05, 5.770731149823405e-05, 5.636859714286402e-05, 5.446967770694755e-05, 5.273939677863382e-05, 4.9779395340010524e-05, 4.799514863407239e-05, 4.710413486463949e-05, 4.5558164856629446e-05, 4.456043097889051e-05, 4.237746907165274e-05, 4.111218368052505e-05, 3.936788561986759e-05, 3.890958760166541e-05, 3.808674227911979e-05, 3.7068453821120784e-05, 3.679451037896797e-05, 3.497559009701945e-05, 3.2898435165407136e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021103879953443538, "first_to_last_ratio": 1.6754332028061545, "imbalance_abs_log_ratio": 0.5160717603794217, "last_quartile_mean": 0.00012596073611348402, "mean": 0.0001601102544555033, "normalized": [ 1.5150372860566723, 1.439510593902249, 1.3841055212355642, 1.3272119947749268, 1.2794362361966427, 1.2428436489609036, 1.2018556832108258, 1.1546727794959446, 1.1147996317367452, 1.0814138049096294, 1.0614905005088058, 1.0346901524019803, 1.0074919017055521, 0.9875011592687566, 0.9623166883068708, 0.9431047552100263, 0.9243559814545742, 0.9073018837351503, 0.8887802361951885, 0.87382889642758, 0.8622859040817829, 0.849391060863955, 0.8374811583413132, 0.8253926631762389, 0.8155827011133436, 0.8058320881575772, 0.7999867009830005, 0.7910419816457831, 0.7825317922950641, 0.7756782882808636, 0.7662211489344684, 0.7568251764320209 ], "peak_layer": 1, "peak_normalized": 1.5150372860566723, "population_cv": 0.21304026877727222, "rest_mean": 0.00016491277050658095, "spike_contrast": 0.8136220583009953, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013417666777968407, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024257300538010895, 0.00023048040748108178, 0.0002216094871982932, 0.00021250025019980967, 0.0002048508613370359, 0.00019899201288353652, 0.0001924294192576781, 0.00018487495253793895, 0.00017849085270427167, 0.00017314543947577477, 0.00016995551413856447, 0.00016566450358368456, 0.00016130978474393487, 0.00015810906188562512, 0.00015407676983159035, 0.00015100074233487248, 0.00014799887139815837, 0.0001452683354727924, 0.00014230282977223396, 0.00013990896695759147, 0.00013806081551592797, 0.00013599621888715774, 0.00013408932136371732, 0.00013215382932685316, 0.00013058315380476415, 0.00012902198068331927, 0.00012808607425540686, 0.0001266539329662919, 0.00012529136438388377, 0.00012419404811225832, 0.00012267986312508583, 0.00012117547157686204 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016164849512279034, "first_to_last_ratio": 1.282986844747637, "imbalance_abs_log_ratio": 0.24919083207168544, "last_quartile_mean": 0.00012599388355738483, "mean": 0.00014267631695474847, "normalized": [ 1.1849632230478688, 1.1695181917643926, 1.1537907432295644, 1.136648454640294, 1.1262413432306055, 1.112705815659591, 1.0977204646042988, 1.082200264843305, 1.06813080649915, 1.0535985061590996, 1.046252801833764, 1.0365714691025223, 1.024670983408149, 1.0169605538784676, 1.0046472025165352, 0.995758555548206, 0.9858673222379531, 0.9760103584099877, 0.9653106304982666, 0.9567951741123486, 0.9498417329681972, 0.9401737612553472, 0.9304537733640251, 0.9205681496416466, 0.911480005330001, 0.9022600866995591, 0.8969925816929462, 0.8873573495747816, 0.8789659160771423, 0.8714788501373707, 0.8627683840784355, 0.8532965439561776 ], "peak_layer": 1, "peak_normalized": 1.1849632230478688, "population_cv": 0.09608600673281108, "rest_mean": 0.00014451252193086676, "spike_contrast": 0.918680321330362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013276081008370966, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016906618839129806, 0.00016686254821252078, 0.00016461861378047615, 0.00016217281518038362, 0.00016068796685431153, 0.00015875676763243973, 0.0001566187129355967, 0.00015440434799529612, 0.00015239696949720383, 0.0001503235544078052, 0.00014927549636922777, 0.00014789419947192073, 0.00014619628200307488, 0.0001450961863156408, 0.00014333936269395053, 0.00014207116328179836, 0.00014065991854295135, 0.00013925356324762106, 0.00013772696547675878, 0.0001365120115224272, 0.00013551992014981806, 0.00013414052955340594, 0.00013275371748022735, 0.0001313432730967179, 0.00013004661013837904, 0.00012873114610556513, 0.0001279795978916809, 0.00012660487846005708, 0.0001254076196346432, 0.00012433939264155924, 0.00012309661542531103, 0.00012174520816188306 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00043634913890855387, "first_to_last_ratio": 3.0823770886184128, "imbalance_abs_log_ratio": 1.125701081333854, "last_quartile_mean": 0.00014156254292174708, "mean": 0.0002526328457861382, "normalized": [ 2.161262991989335, 1.9867412804887739, 1.8904526133838582, 1.7788259407438674, 1.6404609794852394, 1.5723911781894129, 1.459249713339863, 1.3282687448916313, 1.2383315143134188, 1.1700416770361293, 1.131363959578303, 1.0685850444189446, 1.0069989316428685, 0.9451411723514572, 0.8866050347634621, 0.8407661253267313, 0.7895843736134068, 0.7479926194272515, 0.7047300318080335, 0.6729704299307414, 0.6547401608613147, 0.6315820650248338, 0.6127131155987415, 0.5974089218754778, 0.5839699651029305, 0.5735248801032955, 0.5690230706448541, 0.5599717580649735, 0.5552403508520525, 0.5513694456095706, 0.5470291494736481, 0.5426627600655782 ], "peak_layer": 1, "peak_normalized": 2.161262991989335, "population_cv": 0.4790543297146424, "rest_mean": 0.0002705939723118381, "spike_contrast": 0.5751893185853856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015564276254735886, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005460060201585293, 0.0005019161035306752, 0.0004775904235430062, 0.00044938985956832767, 0.0004144343256484717, 0.0003972376580350101, 0.000368654407793656, 0.00033556431299075484, 0.00031284321448765695, 0.000295590958558023, 0.0002858196967281401, 0.0002699596807360649, 0.0002544010058045387, 0.00023877370404079556, 0.0002239855530206114, 0.00021240513888187706, 0.00019947494729422033, 0.0001889675040729344, 0.0001780379534466192, 0.00017001443484332412, 0.00016540887008886784, 0.00015955837443470955, 0.00015479145804420114, 0.00015092511603143066, 0.00014752999413758516, 0.00014489122258964926, 0.00014375391765497625, 0.00014146725879982114, 0.00014027194993104786, 0.00013929403212387115, 0.00013819753075949848, 0.00013709443737752736 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0023513585329055786, 0.004408807028084993, 0.005783648230135441, 0.006751363631337881, 0.002628475893288851, 0.004723479505628347, 0.006024772301316261, 0.007445912808179855, 0.003561305580660701, 0.005543973762542009, 0.006855612155050039, 0.007876631803810596, 0.0035210708156228065, 0.0058924350887537, 0.007390674669295549, 0.008790239691734314, 0.004162859171628952, 0.006355749908834696, 0.007784162648022175, 0.00937180407345295, 0.004737259820103645, 0.006999304983764887, 0.008728929795324802, 0.010217384435236454, 0.005580419674515724, 0.008008211851119995, 0.010209310799837112, 0.012040933594107628, 0.005235649179667234, 0.007745372597128153, 0.010180563665926456, 0.011906592175364494 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024914481400628574, "first_to_last_ratio": 5.005367240493075, "imbalance_abs_log_ratio": 1.6105107847992817, "last_quartile_mean": 4.977553135177004e-05, "mean": 0.0001223312358433759, "normalized": [ 2.9462138255793877, 2.588435651553335, 2.2854731426364707, 2.0157214492628985, 1.8168708763555168, 1.6875649797632875, 1.5554268829788211, 1.3974214212146046, 1.275680469442624, 1.1684152736282016, 1.0925056707858531, 1.0000365768115411, 0.9220243844233116, 0.8633153808206192, 0.7954940059538663, 0.747131175135411, 0.6955592946052511, 0.6567700236831091, 0.6067846463225012, 0.5734077695641915, 0.5496986650820618, 0.5277889932643854, 0.50090777460584, 0.47622023530363405, 0.457550957846772, 0.43796965221258716, 0.42825768814458814, 0.4157111150450942, 0.4005890169587395, 0.39033503431894295, 0.37039430607646523, 0.3543236606200867 ], "peak_layer": 1, "peak_normalized": 2.9462138255793877, "population_cv": 0.6883836085455175, "rest_mean": 0.00013360307773971862, "spike_contrast": 0.46004396487681354, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.14632896031253e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036041397834196687, 0.00031664653215557337, 0.0002795847540255636, 0.0002465856960043311, 0.0002222600596724078, 0.0002064419095404446, 0.00019027729285880923, 0.00017094828945118934, 0.00015605556836817414, 0.00014293368440121412, 0.00013364756887312979, 0.00012233571032993495, 0.00011279238242423162, 0.00010561043745838106, 9.73137648543343e-05, 9.139747999142855e-05, 8.508862811140716e-05, 8.0343488662038e-05, 7.422871567541733e-05, 7.014568109298125e-05, 6.724531704094261e-05, 6.456507981056347e-05, 6.127666711108759e-05, 5.8256609918316826e-05, 5.597277413471602e-05, 5.357736881705932e-05, 5.238929225015454e-05, 5.0854454457294196e-05, 4.9004549509845674e-05, 4.775016714120284e-05, 4.531079321168363e-05, 4.334485129220411e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00029868123783671763, "first_to_last_ratio": 2.352020548935129, "imbalance_abs_log_ratio": 0.8552747667903616, "last_quartile_mean": 0.00012698921273113228, "mean": 0.0001871975605354237, "normalized": [ 2.1745956556501205, 1.943389916860048, 1.7490167059616495, 1.579801534641211, 1.4557872843351107, 1.3727714128797337, 1.290211227528517, 1.198748061313539, 1.1282731192017241, 1.0677025479901412, 1.0238711728275764, 0.9743826669000581, 0.9332092274716193, 0.901273964826617, 0.8680163418407979, 0.8423074677807364, 0.8168973316425937, 0.7977426572048737, 0.7748904888155362, 0.7599280177062464, 0.7477886704872306, 0.7367391764207782, 0.7235805599766263, 0.7121146362854748, 0.7033711688788506, 0.6946931548498627, 0.6884857337520279, 0.6826539310873735, 0.6746941940291191, 0.6690858034871798, 0.660454275348722, 0.6535218920183048 ], "peak_layer": 1, "peak_normalized": 2.1745956556501205, "population_cv": 0.39609817473730136, "rest_mean": 0.00019674051630621156, "spike_contrast": 0.6895661448911525, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013566559937316924, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004070790018886328, 0.00036379785160534084, 0.00032741166069172323, 0.00029573499341495335, 0.00027251982828602195, 0.00025697945966385305, 0.00024152439436875284, 0.000224402712774463, 0.00021120997553225607, 0.00019987131236121058, 0.00019166618585586548, 0.0001824020582716912, 0.00017469449085183442, 0.00016871628758963197, 0.00016249054169747978, 0.00015767790318932384, 0.00015292118769139051, 0.0001493354793637991, 0.0001450576091883704, 0.00014225667109712958, 0.00013998421491123736, 0.00013791577657684684, 0.00013545251567848027, 0.00013330612273421139, 0.00013166936696507037, 0.00013004486390855163, 0.00012888284982182086, 0.00012779115058947355, 0.00012630110722966492, 0.00012525123020168394, 0.00012363542919047177, 0.00012233770394232124 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.2648735139518976, "first_to_last_ratio": 2.3520205903150884, "imbalance_abs_log_ratio": 0.855274784383728, "last_quartile_mean": 0.1126153040677309, "mean": 0.1660086705815047, "normalized": [ 2.174595632268317, 1.943389981095013, 1.7490168198184282, 1.5798016086666014, 1.4557873374041121, 1.3727712901980273, 1.2902112306135503, 1.1987480127171795, 1.1282731510481394, 1.0677025559476454, 1.0238711946273023, 0.9743826246747078, 0.9332091864770874, 0.9012739856841682, 0.8680163310272434, 0.8423074257801569, 0.8168973360433722, 0.7977426245274992, 0.7748904637775177, 0.7599280557788093, 0.7477886868929706, 0.7367391104705062, 0.7235805912981054, 0.7121146568868953, 0.7033711391232357, 0.6946931471398464, 0.6884857463566807, 0.682653951912613, 0.6746941853139277, 0.6690857619715078, 0.6604542663637077, 0.6535219080951254 ], "peak_layer": 1, "peak_normalized": 2.174595632268317, "population_cv": 0.39609818748801673, "rest_mean": 0.1744714594549603, "spike_contrast": 0.6895661390160055, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12030961066484451, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.36100172996520996, 0.32261958718299866, 0.2903519570827484, 0.262260764837265, 0.2416733205318451, 0.2278919368982315, 0.21418625116348267, 0.19900256395339966, 0.18730312585830688, 0.17724788188934326, 0.1699714958667755, 0.16175596415996552, 0.1549208164215088, 0.14961929619312286, 0.14409823715686798, 0.1398303359746933, 0.13561204075813293, 0.13243219256401062, 0.12863853573799133, 0.12615464627742767, 0.12413940578699112, 0.12230508029460907, 0.12012065201997757, 0.11821720749139786, 0.11676570773124695, 0.11532508581876755, 0.11429460346698761, 0.11332647502422333, 0.1120050847530365, 0.1110740378499031, 0.10964113473892212, 0.10849030315876007 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010120356273546349, "first_to_last_ratio": 2.7021644745482813, "imbalance_abs_log_ratio": 0.9940531090948486, "last_quartile_mean": 3.7452776723512216e-05, "mean": 6.280785908074904e-05, "normalized": [ 1.9944873163456722, 1.8472744197374709, 1.7373237819064158, 1.6248552595134786, 1.5471235814302835, 1.4758134067763151, 1.3807453375978929, 1.2829373095670376, 1.18406084586495, 1.104394561286083, 1.0901943179184503, 1.04561090576924, 0.9888677388041794, 0.9664019181449638, 0.9198712145053942, 0.9041143079769479, 0.8692998477179782, 0.8446721332610243, 0.7923141515826202, 0.7644051939108963, 0.7503405467650113, 0.7300062685920449, 0.7090277780404898, 0.6754004942706008, 0.6572075474610624, 0.6267776932652476, 0.618543683654507, 0.6069002486680706, 0.5945805123210514, 0.585842809953234, 0.5555319861870183, 0.5250728812043677 ], "peak_layer": 1, "peak_normalized": 1.9944873163456722, "population_cv": 0.404835319326031, "rest_mean": 6.624604820562699e-05, "spike_contrast": 0.6678381428743124, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.424163780640811e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001252694783033803, 0.00011602335143834352, 0.00010911758727161214, 0.00010205368016613647, 9.717151988297701e-05, 9.269268048228696e-05, 8.672165859024972e-05, 8.057854574872181e-05, 7.436832675011829e-05, 6.936465797480196e-05, 6.847277109045535e-05, 6.567258242284879e-05, 6.210866558831185e-05, 6.069763549021445e-05, 5.7775141613092273e-05, 5.678548404830508e-05, 5.459886233438738e-05, 5.305204831529409e-05, 4.9763555580284446e-05, 4.801065369974822e-05, 4.7127283323789015e-05, 4.585013084579259e-05, 4.4532516767503694e-05, 4.242045906721614e-05, 4.127779902773909e-05, 3.936656503356062e-05, 3.884940451825969e-05, 3.811810529441573e-05, 3.734432903002016e-05, 3.679553265101276e-05, 3.489177470328286e-05, 3.297870352980681e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0002129299391526729, "first_to_last_ratio": 1.6819035009332732, "imbalance_abs_log_ratio": 0.5199261882838202, "last_quartile_mean": 0.00012660056836466538, "mean": 0.0001612697526525153, "normalized": [ 1.5193801883639617, 1.4431946098051458, 1.385412670946279, 1.330312377973009, 1.2819779644638467, 1.2449644115522398, 1.2025034693572292, 1.15492638462487, 1.1161420015875916, 1.082061394531008, 1.0617850512168836, 1.0341029835254725, 1.0077924632434987, 0.9864749169321705, 0.9634524108641764, 0.9447429702319522, 0.9240217763435827, 0.9078672036949645, 0.8863448276060427, 0.8724873261680857, 0.860780266694715, 0.8496270597212868, 0.835986840267667, 0.8234693045553775, 0.8139348844586144, 0.8040606454444867, 0.7969452920911189, 0.7902469974302787, 0.780967396404848, 0.7743340697791917, 0.7639952190801171, 0.7557046210402876 ], "peak_layer": 1, "peak_normalized": 1.5193801883639617, "population_cv": 0.21463676643173402, "rest_mean": 0.00016614488736700473, "spike_contrast": 0.8122069076744362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001349440251942724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024503006716258824, 0.00023274363775271922, 0.000223425158765167, 0.0002145391481462866, 0.00020674426923505962, 0.000200775102712214, 0.00019392743706703186, 0.00018625469238031656, 0.00017999994452111423, 0.00017450377345085144, 0.00017123381257988513, 0.00016676953237038106, 0.00016252644127234817, 0.00015908856585156173, 0.0001553757319925353, 0.00015235846512950957, 0.0001490167633164674, 0.00014641151938121766, 0.00014294061111286283, 0.00014070581528358161, 0.00013881782069802284, 0.00013701914576813579, 0.00013481939095072448, 0.00013280069106258452, 0.00013126307749189436, 0.00012967066140845418, 0.0001285231701331213, 0.00012744293780997396, 0.0001259464188478887, 0.00012487666390370578, 0.00012320932000875473, 0.00012187229731353 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016320142822223715, "first_to_last_ratio": 1.2889451135618213, "imbalance_abs_log_ratio": 0.25382414241573026, "last_quartile_mean": 0.0001266162744286703, "mean": 0.00014373415069712792, "normalized": [ 1.1884743462891088, 1.1713432082551107, 1.1549462763142666, 1.141829986160868, 1.1273728485863819, 1.1144091299418613, 1.0997826147068885, 1.0853573683237072, 1.068690221812727, 1.0554384700276902, 1.0473609881004726, 1.0364070219594264, 1.025017513262844, 1.015761575885501, 1.006046000385858, 0.9971413723222552, 0.9860728003762654, 0.9774913363845525, 0.9641287247440273, 0.9550391277708558, 0.9473133601248869, 0.9403037775179377, 0.9293980025104189, 0.9176260020019316, 0.9094819029195437, 0.8991813518536786, 0.8931890476284186, 0.8860794380468366, 0.8769637206693581, 0.8703930215998912, 0.8602546600212342, 0.8517047834951949 ], "peak_layer": 1, "peak_normalized": 1.1884743462891088, "population_cv": 0.09781347741247146, "rest_mean": 0.00014562865706264146, "spike_contrast": 0.9167413819240867, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013350381632335485, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017082435078918934, 0.00016836202121339738, 0.00016600522212684155, 0.0001641199633013457, 0.00016204197891056538, 0.00016017864982131869, 0.0001580763200763613, 0.0001560029195388779, 0.00015360728139057755, 0.00015170255210250616, 0.00015054154209792614, 0.00014896708307787776, 0.00014733002171851695, 0.00014599962742067873, 0.00014460316742770374, 0.00014332326827570796, 0.00014173233648762107, 0.00014049888704903424, 0.00013857822341378778, 0.00013727173791266978, 0.0001361612812615931, 0.0001351537648588419, 0.00013358623255044222, 0.00013189419405534863, 0.00013072360889054835, 0.00012924306793138385, 0.00012838176917284727, 0.00012735987547785044, 0.00012604963558260351, 0.00012510520173236728, 0.00012364797294139862, 0.00012241906370036304 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00044128536683274433, "first_to_last_ratio": 3.0989416798064484, "imbalance_abs_log_ratio": 1.1310606595922121, "last_quartile_mean": 0.0001423987323505571, "mean": 0.0002548205006860371, "normalized": [ 2.1788974669107217, 1.9948922239742408, 1.9013861017158324, 1.7832746019711707, 1.6474456862757656, 1.56528080717771, 1.458232784418858, 1.3245892215128081, 1.2376166576486232, 1.160533346247024, 1.1299807802432356, 1.0675072615016112, 1.0048234765069712, 0.9484981041710848, 0.8855756138701967, 0.8439855543529297, 0.7905268728467926, 0.7470509271970999, 0.6989608308202429, 0.6713105901225432, 0.6509228727076454, 0.631461080612247, 0.611226351598172, 0.5954626643405772, 0.5833123929329473, 0.5733046444152264, 0.5652547933675338, 0.559916360487464, 0.5532517426009346, 0.5496220513891801, 0.5450027610740823, 0.5408933749885276 ], "peak_layer": 1, "peak_normalized": 2.1788974669107217, "population_cv": 0.4822174837001359, "rest_mean": 0.0002730129350244102, "spike_contrast": 0.5735309033794403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015658135525882245, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005552277434617281, 0.0005083394353277981, 0.0004845121584367007, 0.0004544149269349873, 0.00041980293462984264, 0.0003988656389992684, 0.00037158760824240744, 0.0003375324886292219, 0.00031537009635940194, 0.0002957276883535087, 0.00028794226818718016, 0.00027202273486182094, 0.00025604962138459086, 0.00024169676180463284, 0.0002256628213217482, 0.00021506482153199613, 0.0002014424535445869, 0.00019036389130633324, 0.00017810954886954278, 0.00017106370069086552, 0.0001658684923313558, 0.00016090922872535884, 0.00015575300494674593, 0.00015173609426710755, 0.00014863995602354407, 0.00014608977653551847, 0.00014403850946109742, 0.00014267816732171923, 0.00014097988605499268, 0.00014005496632307768, 0.0001388778764521703, 0.00013783072063233703 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0036361459642648697, 0.005152894649654627, 0.006386642809957266, 0.007284422870725393, 0.0037505170330405235, 0.0054016136564314365, 0.006585344672203064, 0.007922882214188576, 0.0044892593286931515, 0.006135277915745974, 0.007316684816032648, 0.0082859443500638, 0.0044554853811860085, 0.006464940495789051, 0.007893112488090992, 0.009196000173687935, 0.004961193539202213, 0.006907128728926182, 0.008217990398406982, 0.009676996618509293, 0.005381465889513493, 0.007502692751586437, 0.0091153085231781, 0.010543142445385456, 0.006136673036962748, 0.008469858206808567, 0.010564134456217289, 0.012301022186875343, 0.005773921962827444, 0.008156762458384037, 0.010457510128617287, 0.012218860909342766 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024784039669611957, "first_to_last_ratio": 4.996847129630424, "imbalance_abs_log_ratio": 1.6088071394647374, "last_quartile_mean": 4.959935540682636e-05, "mean": 0.0001217503483985638, "normalized": [ 2.9419062071052267, 2.5796112881191693, 2.2930860326581057, 2.0145554032379356, 1.8111535572920587, 1.6841756414503508, 1.5558994713918, 1.4047664015764878, 1.2749699043555138, 1.1699192416770665, 1.0919281315608147, 1.001760718810264, 0.9234449853785138, 0.8652188455092179, 0.7945394094365259, 0.7439156225148, 0.6942882953063699, 0.6562804150522287, 0.6082998698912725, 0.574440737760942, 0.551250788347098, 0.5265559301830911, 0.5020487822241618, 0.47689842352430634, 0.4572026867293029, 0.43893712719402944, 0.4296155316374904, 0.41603615348749273, 0.3999129464790325, 0.391194073390914, 0.3717285968379238, 0.3544587798804922 ], "peak_layer": 1, "peak_normalized": 2.9419062071052267, "population_cv": 0.6874423943830519, "rest_mean": 0.00013296059464833147, "spike_contrast": 0.4603997057303065, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.121501864981837e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00035817810567095876, 0.0003140685730613768, 0.00027918402338400483, 0.00024527282221242785, 0.00022050857660360634, 0.00020504897111095488, 0.0001894313027150929, 0.00017103079881053418, 0.00015522803005296737, 0.0001424380752723664, 0.000132942630443722, 0.00012196471652714536, 0.00011242974869674072, 0.00010534069588175043, 9.673544991528615e-05, 9.057198622031137e-05, 8.452984184259549e-05, 7.99023691797629e-05, 7.406072109006345e-05, 6.99383599567227e-05, 6.711497553624213e-05, 6.410836795112118e-05, 6.112461414886639e-05, 5.806254921481013e-05, 5.566458639805205e-05, 5.34407481609378e-05, 5.230584065429866e-05, 5.0652546633500606e-05, 4.868954056291841e-05, 4.762801472679712e-05, 4.5258086174726486e-05, 4.315547994337976e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00029608183103846386, "first_to_last_ratio": 2.343484699021421, "imbalance_abs_log_ratio": 0.8516390092453431, "last_quartile_mean": 0.0001263425492652459, "mean": 0.0001858347804954974, "normalized": [ 2.16920774571012, 1.9356424765570717, 1.751272112160408, 1.5771369240348272, 1.451727119925344, 1.369130935687747, 1.289826251583108, 1.2020808437231323, 1.127478492836466, 1.067970341065492, 1.0234512239081746, 0.9751296621832141, 0.9332398141881068, 0.9026389800429313, 0.8668613586828855, 0.8407822065969367, 0.8171794710239251, 0.7975274126338607, 0.777066300159633, 0.7612061150730458, 0.7492407752334779, 0.7365489154759011, 0.7249215428669998, 0.7138135717041164, 0.7048334785593375, 0.6962720074755282, 0.6911692191098191, 0.683337870004401, 0.6760758813563923, 0.6702804011180051, 0.662430102042749, 0.6545204472768429 ], "peak_layer": 1, "peak_normalized": 2.16920774571012, "population_cv": 0.39438035124793563, "rest_mean": 0.0001952685916977417, "spike_contrast": 0.6908033638721538, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001348922000033781, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00040311424527317286, 0.00035970969474874437, 0.00032544726855121553, 0.00029308689408935606, 0.00026978139067068696, 0.00025443214690312743, 0.00023969457834027708, 0.0002233884297311306, 0.0002095247182296589, 0.0001984660339076072, 0.0001901928335428238, 0.00018121300672646612, 0.0001734284160193056, 0.0001677417167229578, 0.00016109299031086266, 0.00015624657680746168, 0.0001518603676231578, 0.00014820833166595548, 0.00014440594532061368, 0.0001414585713064298, 0.00013923499500378966, 0.00013687640603166074, 0.00013471563579514623, 0.00013265138841234148, 0.00013098257477395236, 0.0001293915556743741, 0.00012844328011851758, 0.0001269879430765286, 0.0001256384130101651, 0.00012456141121219844, 0.0001231025526067242, 0.00012163266364950687 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.2625683266669512, "first_to_last_ratio": 2.343484691137257, "imbalance_abs_log_ratio": 0.8516390058810522, "last_quartile_mean": 0.1120418356731534, "mean": 0.16480014030821621, "normalized": [ 2.169207748902955, 1.9356423730461585, 1.7512721262865893, 1.5771370154883375, 1.4517271231757105, 1.3691309069442503, 1.2898263254837699, 1.202080803995401, 1.1274784933366124, 1.0679703770106845, 1.0234512066143264, 0.9751297116000902, 0.9332398524746174, 0.9026389696372329, 0.8668612939874407, 0.8407822044491968, 0.8171795266543764, 0.7975273711861923, 0.7770662316421936, 0.7612060929585753, 0.7492407781338722, 0.7365489418998509, 0.7249215250846839, 0.7138135688166896, 0.704833501980704, 0.6962720326387369, 0.6911692482611179, 0.6833378266924969, 0.6760758677045711, 0.6702804240542598, 0.6624300595814382, 0.6545204702768674 ], "peak_layer": 1, "peak_normalized": 2.169207748902955, "population_cv": 0.3943803506949354, "rest_mean": 0.17316613815448903, "spike_contrast": 0.6908033707584418, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11962375193834304, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.35748574137687683, 0.3189941346645355, 0.28860989212989807, 0.2599124014377594, 0.23924483358860016, 0.22563296556472778, 0.21256355941295624, 0.19810308516025543, 0.18580861389636993, 0.1760016679763794, 0.16866490244865417, 0.16070151329040527, 0.15379805862903595, 0.1487550288438797, 0.1428588628768921, 0.13856102526187897, 0.13467130064964294, 0.13143262267112732, 0.12806062400341034, 0.1254468709230423, 0.12347498536109924, 0.12138336896896362, 0.11946716904640198, 0.11763657629489899, 0.11615666002035141, 0.11474572867155075, 0.11390478909015656, 0.11261416971683502, 0.11141739785671234, 0.11046230792999268, 0.10916856676340103, 0.10786506533622742 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010104048214998329, "first_to_last_ratio": 2.7014816335786884, "imbalance_abs_log_ratio": 0.9938003756776099, "last_quartile_mean": 3.740187639778014e-05, "mean": 6.270627807225537e-05, "normalized": [ 1.9907250147127677, 1.8444603036108247, 1.7429056467158477, 1.6239117898045463, 1.5436487234069576, 1.4786997242989472, 1.3841625886805662, 1.282123032540977, 1.1824417181452251, 1.1042125830371328, 1.090862943651666, 1.0498276343781445, 0.9920003536086259, 0.9660066676365643, 0.9202805760437535, 0.8989382213875909, 0.8686500616317681, 0.8410510099026279, 0.7938571089189717, 0.7653969230084752, 0.751185801344563, 0.7265337395686793, 0.7106236720469997, 0.6758027080339929, 0.6556269427545545, 0.6278192648368973, 0.6205119527339269, 0.6073894466988616, 0.5911483388457922, 0.586775311152339, 0.5577767930941091, 0.5246434037673043 ], "peak_layer": 1, "peak_normalized": 1.9907250147127677, "population_cv": 0.4046861823965828, "rest_mean": 6.614403860617636e-05, "spike_contrast": 0.6673673413246971, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.414237118908204e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012483095633797348, 0.00011565924069145694, 0.00010929112613666803, 0.0001018294642562978, 9.67964660958387e-05, 9.272375609725714e-05, 8.679568418301642e-05, 8.039716340135783e-05, 7.41465191822499e-05, 6.924106128280982e-05, 6.84039550833404e-05, 6.583078356925398e-05, 6.220465002115816e-05, 6.0574682720471174e-05, 5.770736970589496e-05, 5.636907008010894e-05, 5.446981231216341e-05, 5.273917849990539e-05, 4.977982462150976e-05, 4.799519228981808e-05, 4.7104065743042156e-05, 4.555822670226917e-05, 4.456056558410637e-05, 4.237707253196277e-05, 4.1111925384029746e-05, 3.9368209399981424e-05, 3.89099950552918e-05, 3.808713154285215e-05, 3.706871211761609e-05, 3.679449582705274e-05, 3.4976106690010056e-05, 3.2898435165407136e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021103927974763792, "first_to_last_ratio": 1.675431813331652, "imbalance_abs_log_ratio": 0.5160709310565479, "last_quartile_mean": 0.00012596112719620578, "mean": 0.00016011051866371417, "normalized": [ 1.5150472374771589, 1.439502583502474, 1.3841101446264577, 1.327205260326266, 1.2794452130965426, 1.2428385988073838, 1.2018548814818522, 1.154676418187547, 1.1148050630726085, 1.0814152014364051, 1.0614838409967562, 1.034687172582471, 1.0074911480481314, 0.9875049829338368, 0.9623163727426111, 0.9431041078017948, 0.9243534563643369, 0.9073040220081667, 0.888778951337658, 0.8738270000353379, 0.8622815727966864, 0.8493922949455803, 0.8374788674952537, 0.8253865750371285, 0.8155793557629833, 0.8058361207192702, 0.7999903796437624, 0.7910448570865577, 0.7825332275919701, 0.7756749178925374, 0.766226246614208, 0.7568239275482656 ], "peak_layer": 1, "peak_normalized": 1.5150472374771589, "population_cv": 0.21304076077436968, "rest_mean": 0.0001649131305317429, "spike_contrast": 0.8136187466924137, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001341764145763591, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024257499899249524, 0.00023047950526233763, 0.00022161059314385056, 0.00021249952260404825, 0.00020485263667069376, 0.00019899153267033398, 0.00019242960843257606, 0.00018487584020476788, 0.00017849201685748994, 0.00017314594879280776, 0.00016995472833514214, 0.00016566429985687137, 0.00016130993026308715, 0.00015810993500053883, 0.00015407697355840355, 0.00015100088785402477, 0.00014799871132709086, 0.00014526891754940152, 0.00014230285887606442, 0.00013990889419801533, 0.00013806034985464066, 0.00013599664089269936, 0.00013408917584456503, 0.00013215307262726128, 0.00013058283366262913, 0.00012902283924631774, 0.00012808687461074442, 0.0001266546023543924, 0.00012529180094134063, 0.00012419371341820806, 0.00012268088175915182, 0.00012117547157686204 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016164389126060996, "first_to_last_ratio": 1.2829777917904304, "imbalance_abs_log_ratio": 0.24918377588932214, "last_quartile_mean": 0.00012599118417710997, "mean": 0.0001426773092134681, "normalized": [ 1.1847160153617624, 1.1696203114063919, 1.15376099486318, 1.1365907777467705, 1.126248503505431, 1.112765799827437, 1.097614510342432, 1.0821504120145113, 1.06828860482523, 1.0536786878013262, 1.046275103221197, 1.0365937358276223, 1.0246881313142864, 1.0168969779036778, 1.0047039605050023, 0.9958410772863212, 0.9859208451030141, 0.9762312163738422, 0.965237112554715, 0.9568250330809598, 0.9498426746244831, 0.9400597234006037, 0.9304691287067991, 0.9205814319068928, 0.9114724424808572, 0.9022957305012854, 0.8969570718532309, 0.8872712168205663, 0.8789155388108255, 0.8714969614364647, 0.8626996589435726, 0.8532906096493081 ], "peak_layer": 1, "peak_normalized": 1.1847160153617624, "population_cv": 0.0960837578275494, "rest_mean": 0.00014451401323269778, "spike_contrast": 0.9186590596986468, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013275910750962794, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000169032093253918, 0.00016687827883288264, 0.0001646155142225325, 0.00016216571384575218, 0.0001606901059858501, 0.00015876643010415137, 0.00015660468488931656, 0.00015439830895047635, 0.00015242054359987378, 0.00015033603995107114, 0.000149279716424644, 0.00014789840497542173, 0.00014619974535889924, 0.0001450881245546043, 0.00014334845764096826, 0.00014208392531145364, 0.00014066853327676654, 0.00013928604312241077, 0.0001377174339722842, 0.00013651722110807896, 0.00013552099699154496, 0.00013412519183475524, 0.00013275683159008622, 0.00013134608161635697, 0.0001300464355153963, 0.000128737126942724, 0.00012797542149201035, 0.00012659346975851804, 0.00012540130410343409, 0.00012434284144546837, 0.00012308766599744558, 0.00012174520816188306 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00043632716551655903, "first_to_last_ratio": 3.082278345441195, "imbalance_abs_log_ratio": 1.1256690460714511, "last_quartile_mean": 0.00014155994904285762, "mean": 0.00025262987765017897, "normalized": [ 2.1611427674158366, 1.9871567751170711, 1.8899978819847916, 1.7784604477508343, 1.6406551319992353, 1.5724690971097588, 1.4592296472822097, 1.3280082080145534, 1.2379389344592089, 1.1701308820499392, 1.131654892203279, 1.0685236385646504, 1.0072825277339317, 0.9454673581714634, 0.8865158004683366, 0.8408935109532261, 0.7895809204213243, 0.7481431077843942, 0.7047226439788088, 0.6727720648712864, 0.6548656488865329, 0.6315662143568483, 0.6127430670138975, 0.5973169234484098, 0.5839093169225268, 0.5733930286828336, 0.5691911560855093, 0.5599482690481828, 0.5552538441515218, 0.5513421690146183, 0.547054988271426, 0.542669135783553 ], "peak_layer": 1, "peak_normalized": 2.1611427674158366, "population_cv": 0.47902914554115894, "rest_mean": 0.0002705909153317205, "spike_contrast": 0.5751866206559582, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001556402741698548, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000545969232916832, 0.0005020151729695499, 0.0004774699336849153, 0.0004492922453209758, 0.00041447850526310503, 0.0003972526756115258, 0.00036864500725641847, 0.00033549455110915005, 0.00031274036155082285, 0.00029561002156697214, 0.00028588983695954084, 0.0002699409960769117, 0.00025446966174058616, 0.00023885330301709473, 0.00022396037820726633, 0.00021243482478894293, 0.00019947173132095486, 0.00018900330178439617, 0.0001780339953256771, 0.00016996232443489134, 0.00016543862875550985, 0.0001595524954609573, 0.00015479720605071634, 0.0001509001012891531, 0.00014751293929293752, 0.0001448562106816098, 0.00014379469212144613, 0.00014145966270007193, 0.0001402737107127905, 0.00013928550470154732, 0.00013820243475493044, 0.00013709443737752736 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.027978340163826942, 0.0140878576785326, 0.014187770895659924, 0.014337530359625816, 0.014477464370429516, 0.009757626801729202, 0.009825476445257664, 0.00987118761986494, 0.009999874979257584, 0.007595349568873644, 0.007658981718122959, 0.007689184043556452, 0.007755559403449297, 0.00625597033649683, 0.006315375678241253, 0.00635211868211627, 0.006408242508769035, 0.005408597644418478, 0.005475745070725679, 0.005517482291907072, 0.005585856735706329, 0.004830927588045597, 0.004899685736745596, 0.004948681220412254, 0.004989070352166891, 0.004419817589223385, 0.004463871940970421, 0.004542885348200798, 0.004592607729136944, 0.004137600306421518, 0.004193917382508516, 0.004230021964758635 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024312340610777028, "first_to_last_ratio": 2.286272500967691, "imbalance_abs_log_ratio": 0.8269227625411849, "last_quartile_mean": 0.00010634051977831405, "mean": 0.00017363850702167838, "normalized": [ 1.061084599304074, 1.6344608012282997, 1.6065574901355, 1.6039206242202213, 1.1359733847422597, 1.548756457352017, 1.2967221414405534, 1.313883475593909, 1.1957449434685963, 1.344067980756563, 1.165161355052429, 1.1171979507437393, 1.0391889074947966, 1.1329181602378293, 1.0774500297071945, 1.0521010358096583, 0.862761425176938, 0.9019434894037968, 0.9280163978632505, 0.8870730707939211, 0.8354471012965897, 0.8049103603266032, 0.8255366455754076, 0.7297241480789424, 0.6373588767497196, 0.6752584188072907, 0.6298229328646214, 0.6250051046248064, 0.6032481952321226, 0.5674884668477911, 0.6733729976615692, 0.4878430314089892 ], "peak_layer": 2, "peak_normalized": 1.6344608012282997, "population_cv": 0.3218023265359089, "rest_mean": 0.00018114369622371332, "spike_contrast": 0.7348336602687936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001331104853306897, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.00018424514564685524, 0.0002838053333107382, 0.000278960244031623, 0.00027850238257087767, 0.00019724872254300863, 0.00026892375899478793, 0.00022516089666169137, 0.0002281407651025802, 0.0002076273667626083, 0.00023338195751421154, 0.00020231687813065946, 0.00019398858421482146, 0.0001804432104108855, 0.00019671821792144328, 0.00018708681454882026, 0.00018268525309395045, 0.000149808605783619, 0.00015661212091799825, 0.0001611393818166107, 0.00015403004363179207, 0.00014506578736472875, 0.00013976343325339258, 0.00014334495062939823, 0.0001267082116100937, 0.00011067004379583523, 0.00011725086369551718, 0.00010936151375062764, 0.00010852495324797928, 0.00010474711598362774, 9.853785013547167e-05, 0.000116923481982667, 8.470833563478664e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00025287891548941843, "first_to_last_ratio": 2.2739168136310908, "imbalance_abs_log_ratio": 0.8215038131425536, "last_quartile_mean": 0.0001112085165004828, "mean": 0.0001807239029858465, "normalized": [ 1.0592123382591336, 1.645736468094544, 1.60181949161791, 1.5891655894821897, 1.1402836456293184, 1.5483130487587196, 1.300612463539172, 1.3089009659072275, 1.1885078058073624, 1.3408206015264876, 1.166937022076678, 1.1148751106540256, 1.0356001272424655, 1.1351884917697168, 1.0748875173492451, 1.0458945475116934, 0.8650051516349683, 0.9033374117584585, 0.9274894298403746, 0.8851829368231718, 0.8352656795149113, 0.8098299287299446, 0.8239713600606818, 0.7303600865627311, 0.6419462275110265, 0.6820467100618508, 0.635541252420758, 0.625618271589476, 0.6029839777576883, 0.5727133544881818, 0.6681570652306708, 0.4937959207892163 ], "peak_layer": 2, "peak_normalized": 1.645736468094544, "population_cv": 0.3200470231318611, "rest_mean": 0.00018847914530110925, "spike_contrast": 0.7366629038009043, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001388455944834277, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001914249878609553, 0.00029742391780018806, 0.00028948707040399313, 0.0002872002078220248, 0.00020607651094906032, 0.0002798171772155911, 0.00023505176068283617, 0.00023654969118069857, 0.00021479176939465106, 0.0002423183323116973, 0.00021089341316837817, 0.00020148458133917302, 0.00018715769692789763, 0.00020515569485723972, 0.0001942578674061224, 0.0001890181447379291, 0.00015632710710633546, 0.00016325466276612133, 0.00016761950973886997, 0.0001599737151991576, 0.00015095247363206, 0.0001463556254748255, 0.00014891132013872266, 0.00013199352542869747, 0.0001160150277428329, 0.0001232621434610337, 0.00011485749564599246, 0.00011306417582090944, 0.00010897361789830029, 0.0001035029927152209, 0.00012075195263605565, 8.924072608351707e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.22425555251538754, "first_to_last_ratio": 2.2739168285843645, "imbalance_abs_log_ratio": 0.8215038197185521, "last_quartile_mean": 0.09862082451581955, "mean": 0.1602677651681006, "normalized": [ 1.0592124025748084, 1.645736496509649, 1.6018195299533093, 1.5891655927499475, 1.1402836784663197, 1.5483129508886688, 1.300612441604666, 1.3089009387606192, 1.1885077887054891, 1.3408205692725423, 1.1669370180912673, 1.1148751103389594, 1.0356000505037584, 1.1351884646301031, 1.0748875233039092, 1.0458946120331756, 0.8650051766933494, 0.9033374073089909, 0.9274894888798537, 0.8851828780404403, 0.835265755835364, 0.8098299455472033, 0.8239713233255873, 0.730360099613448, 0.6419462245963619, 0.6820467317825786, 0.6355412485786012, 0.6256182682629655, 0.602983984145969, 0.5727133411523797, 0.6681570418174577, 0.49379591603225714 ], "peak_layer": 2, "peak_normalized": 1.645736496509649, "population_cv": 0.3200470194234096, "rest_mean": 0.16714519124340127, "spike_contrast": 0.7366629183018027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1231296643614769, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.16975760459899902, 0.26375851035118103, 0.25672003626823425, 0.254692018032074, 0.18275071680545807, 0.24814465641975403, 0.20844624936580658, 0.20977462828159332, 0.19047948718070984, 0.21489031612873077, 0.1870223879814148, 0.17867854237556458, 0.16597330570220947, 0.18193411827087402, 0.1722698211669922, 0.16762319207191467, 0.13863244652748108, 0.1447758674621582, 0.14864666759967804, 0.1418662816286087, 0.13386617600917816, 0.12978963553905487, 0.13205604255199432, 0.11705318093299866, 0.10288328677415848, 0.1093101054430008, 0.1018567755818367, 0.10026644170284271, 0.09663889557123184, 0.09178748726844788, 0.10708403587341309, 0.07913956791162491 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.143860511860112e-05, "first_to_last_ratio": 0.9636580872932836, "imbalance_abs_log_ratio": 0.037018728516889636, "last_quartile_mean": 7.413273033307632e-05, "mean": 7.686168260079285e-05, "normalized": [ 0.586022200160155, 0.9787081875553917, 0.912570839911241, 1.015732496910769, 0.8201270226617754, 1.1119332047208261, 1.0008210861718638, 1.0096344009845781, 0.9335569648148418, 1.090167719321714, 1.035494250546431, 0.9697380161483544, 1.0399314841024359, 1.0649519568255064, 1.06563466639597, 1.1389203959735625, 0.9952637999761301, 1.0027093290728673, 1.1427249020241628, 1.1090175842590573, 1.0432879501645442, 1.0450477353718262, 1.1125450117027875, 1.0594965280199307, 0.9180884623091344, 0.995296932026832, 0.9631737996002885, 0.9446177686211702, 0.9674620335911367, 0.9355074959711648, 1.1351899163905306, 0.8566258576930204 ], "peak_layer": 19, "peak_normalized": 1.1427249020241628, "population_cv": 0.10846075768175395, "rest_mean": 7.635363915512614e-05, "spike_contrast": 1.0425844542348655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.960511720739305e-05, "top_five_layers": [ 19, 16, 31, 23, 6 ], "top_five_spike_overlap": 1 }, "values": [ 4.504265234572813e-05, 7.522515807067975e-05, 7.014173024799675e-05, 7.807090878486633e-05, 6.303634290816262e-05, 8.546505705453455e-05, 7.692479266552255e-05, 7.760219887131825e-05, 7.175475911935791e-05, 8.379212522413582e-05, 7.958983042044565e-05, 7.453569560311735e-05, 7.993088365765288e-05, 8.185399929061532e-05, 8.190647349692881e-05, 8.753933798288926e-05, 7.64976502978243e-05, 7.706992619205266e-05, 8.78317587194033e-05, 8.52409575600177e-05, 8.018886728677899e-05, 8.032412733882666e-05, 8.551208156859502e-05, 8.143468585330993e-05, 7.056582398945466e-05, 7.650019688298926e-05, 7.403115887427703e-05, 7.260491111082956e-05, 7.436075975419953e-05, 7.190468022599816e-05, 8.725260704522952e-05, 6.584170478163287e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 8.107683152047684e-05, "first_to_last_ratio": 1.03534602173519, "imbalance_abs_log_ratio": 0.034735691361772646, "last_quartile_mean": 7.83089226388256e-05, "mean": 8.487996228723205e-05, "normalized": [ 0.5954853629443996, 1.025163681929567, 0.9520754208460104, 1.0126519094835282, 0.8443534832663374, 1.1548844288399933, 1.0326421147312754, 1.0242956755731092, 0.9710206925084107, 1.15116021353655, 1.072832454869962, 1.0193061391332177, 1.0419196505755806, 1.0947619162703996, 1.1089694127677372, 1.1239932261258834, 0.9835832115239759, 1.0364664516448652, 1.11612142168979, 1.093437276508762, 1.0179007506072324, 1.033774483284387, 1.0848380474290875, 1.0276879808812729, 0.8974590825111308, 0.9692041024722606, 0.9378977551023808, 0.9155031739616566, 0.918412101050971, 0.8874638084461439, 1.0536745962271137, 0.8010599732570078 ], "peak_layer": 6, "peak_normalized": 1.1548844288399933, "population_cv": 0.11200209045145568, "rest_mean": 8.468612051931107e-05, "spike_contrast": 1.0146492401244358, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.592670783400536e-05, "top_five_layers": [ 6, 10, 16, 19, 15 ], "top_five_spike_overlap": 0 }, "values": [ 5.054477514931932e-05, 8.701585466042161e-05, 8.081212581600994e-05, 8.59538558870554e-05, 7.166869181673974e-05, 9.802654676605016e-05, 8.76506237545982e-05, 8.694217831362039e-05, 8.242019976023585e-05, 9.771043551154435e-05, 9.106197830988094e-05, 8.651866664877161e-05, 8.843810064718127e-05, 9.292335016652942e-05, 9.41292819334194e-05, 9.540450264466926e-05, 8.348650590050966e-05, 8.797523332759738e-05, 9.473634418100119e-05, 9.281091479351744e-05, 8.639937732368708e-05, 8.774673915468156e-05, 9.20810125535354e-05, 8.723011706024408e-05, 7.617629307787865e-05, 8.226600766647607e-05, 7.960872608236969e-05, 7.770787487970665e-05, 7.795478450134397e-05, 7.532789459219202e-05, 8.943585999077186e-05, 6.799394031986594e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.1427921979011444e-05, "first_to_last_ratio": 0.40836765899304117, "imbalance_abs_log_ratio": 0.8955873853644011, "last_quartile_mean": 5.2472132665570825e-05, "mean": 3.864622581772892e-05, "normalized": [ 0.2626641640048693, 0.5716512870137274, 0.5157707114648397, 0.4843895386483896, 0.5109475889421058, 0.7512596968779521, 0.675728069412672, 0.6632970161544233, 0.7004288748821966, 0.9130902556041411, 0.8717836320781894, 0.8044820763577291, 0.9241814793896349, 0.9531622930600316, 1.0319070449118461, 1.0273708471578158, 0.9893551980130566, 1.0844069350216101, 1.1887300698220302, 1.2100415764612447, 1.1128447755732793, 1.2346303137099222, 1.3033621630444512, 1.3524689418392815, 1.2055635543985064, 1.318388624044801, 1.3615446320871405, 1.2933406942036905, 1.3753836688642536, 1.3573239762703455, 1.6442457326403273, 1.3062545680454958 ], "peak_layer": 31, "peak_normalized": 1.6442457326403273, "population_cv": 0.32680115045148705, "rest_mean": 3.691592014042221e-05, "spike_contrast": 1.2999777952883038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.798987647518516e-05, "top_five_layers": [ 31, 29, 27, 30, 24 ], "top_five_spike_overlap": 1 }, "values": [ 1.0150978596357163e-05, 2.2092164726927876e-05, 1.99325913854409e-05, 1.8719827494351193e-05, 1.974619590328075e-05, 2.9033351893303916e-05, 2.6114339561900124e-05, 2.5633926270529628e-05, 2.7068932467955165e-05, 3.528749221004546e-05, 3.369114710949361e-05, 3.1090195989236236e-05, 3.571612614905462e-05, 3.683612521854229e-05, 3.9879312680568546e-05, 3.970400575781241e-05, 3.823484439635649e-05, 4.190823528915644e-05, 4.5939930714666843e-05, 4.676354001276195e-05, 4.300725049688481e-05, 4.771380190504715e-05, 5.037002847529948e-05, 5.226782013778575e-05, 4.65904813609086e-05, 5.09507444803603e-05, 5.261856131255627e-05, 4.998273652745411e-05, 5.3153387852944434e-05, 5.24554488947615e-05, 6.354389188345522e-05, 5.048180901212618e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031520502852799837, "first_to_last_ratio": 1.31300997816315, "imbalance_abs_log_ratio": 0.27232219480674863, "last_quartile_mean": 0.00024006293460843153, "mean": 0.00030979480334281106, "normalized": [ 0.607123837738274, 1.0354820546705408, 0.9347765087792952, 1.1085716606000595, 0.9306423427143409, 1.2565833735558338, 1.1285140374487672, 1.138017377238765, 1.0370737734279494, 1.2142990636699202, 1.1645184658462944, 1.1206382096550715, 1.1232577862254296, 1.1815362250572234, 1.1924463988431009, 1.2309950595128432, 1.0374639291264394, 1.065463636492051, 1.1630951913863559, 1.1082935818936954, 1.0270641612904527, 0.9934767629460804, 1.0585185268550952, 0.9428720751183239, 0.8088208526771234, 0.8683362720246933, 0.8018903045940483, 0.8027332776454693, 0.7640102071384416, 0.7171383971343437, 0.8334550561316563, 0.6028915925620211 ], "peak_layer": 6, "peak_normalized": 1.2565833735558338, "population_cv": 0.17765693833110424, "rest_mean": 0.0003117367305094376, "spike_contrast": 0.9601319554288655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00029930839664302764, "top_five_layers": [ 6, 16, 10, 15, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00018808380991686136, 0.00032078695949167013, 0.00028958890470676124, 0.00034342973958700895, 0.0002883081615436822, 0.00038928299909457564, 0.00034960778430104256, 0.0003525518695823848, 0.00032128006569109857, 0.00037618353962898254, 0.00036076176911592484, 0.00034716789377853274, 0.0003479794249869883, 0.00036603378248400986, 0.0003694136976264417, 0.00038135587237775326, 0.0003214009338989854, 0.0003300750977359712, 0.0003603208460845053, 0.000343343592248857, 0.00031817913986742496, 0.00030777393840253353, 0.00032792353886179626, 0.00029209686908870935, 0.0002505684969946742, 0.0002690060646273196, 0.00024842144921422005, 0.0002486825978849083, 0.00023668639187235385, 0.00022216574870981276, 0.000258200045209378, 0.0001867726823547855 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014031016267836094, 0.014151837676763535, 0.014265215955674648, 0.014401593245565891, 0.009722241200506687, 0.009786579757928848, 0.009839320555329323, 0.00996283907443285, 0.007562503684312105, 0.007635751739144325, 0.007660485804080963, 0.007726665120571852, 0.006238248664885759, 0.006290202960371971, 0.006327399518340826, 0.006382675375789404, 0.005390914157032967, 0.0054517327807843685, 0.005509276874363422, 0.0055727302096784115, 0.0048178439028561115, 0.004884926602244377, 0.004938251338899136, 0.004975184798240662, 0.004412420094013214, 0.004455919377505779, 0.004530393984168768, 0.0045819394290447235, 0.004127359017729759, 0.004185030702501535, 0.004221425391733646, 0.004281661007553339 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.6193598993559135e-05, "first_to_last_ratio": 1.5723739147519853, "imbalance_abs_log_ratio": 0.4525865249799748, "last_quartile_mean": 3.573806361600873e-05, "mean": 4.4294749727669114e-05, "normalized": [ 1.5642840408451666, 1.493254721227763, 1.1816235069560812, 1.0717429494942774, 1.3553344243310417, 1.3257825682631335, 1.1255110934944355, 1.0314977817441078, 1.194075821513187, 1.0893775754580661, 1.0407197138445958, 0.9875719064328349, 1.135803276632048, 1.0324555952346264, 0.9635988909717662, 0.90208003019059, 0.9904506854292476, 0.9196323607404276, 0.8708711781381843, 0.8075366439875266, 0.9218591005239072, 0.8776357461813653, 0.8517186042296377, 0.8109904231766266, 0.8722342267309339, 0.8500552841464633, 0.7844009556175089, 0.7665260139039981, 0.8384393111065888, 0.8154485842192296, 0.7754690286134897, 0.7520179566211438 ], "peak_layer": 1, "peak_normalized": 1.5642840408451666, "population_cv": 0.20749976292329902, "rest_mean": 4.538663482957485e-05, "spike_contrast": 0.8460325450777163, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.839857017737813e-05, "top_five_layers": [ 1, 2, 5, 6, 9 ], "top_five_spike_overlap": 0 }, "values": [ 6.928957009222358e-05, 6.614334415644407e-05, 5.23397175129503e-05, 4.747258572024293e-05, 6.003419912303798e-05, 5.872520705452189e-05, 4.985423220205121e-05, 4.5689936087001115e-05, 5.289128966978751e-05, 4.8253707063850015e-05, 4.6098419261397794e-05, 4.374425043351948e-05, 5.03101218782831e-05, 4.573236219584942e-05, 4.2682371713453904e-05, 3.9957409171620384e-05, 4.3871765228686854e-05, 4.073488526046276e-05, 3.857502088067122e-05, 3.576963354134932e-05, 4.083351814188063e-05, 3.887465572915971e-05, 3.7726662412751466e-05, 3.592261782614514e-05, 3.863539677695371e-05, 3.7652986065950245e-05, 3.474484401522204e-05, 3.395307794562541e-05, 3.713845944730565e-05, 3.612009095377289e-05, 3.4349206543993205e-05, 3.3310447179246694e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 9.100913848669734e-05, "first_to_last_ratio": 0.8419197805282016, "imbalance_abs_log_ratio": 0.17207054181035333, "last_quartile_mean": 0.00010809716150106397, "mean": 9.889523016681778e-05, "normalized": [ 1.0744641570554045, 0.9868986027413778, 0.8415148680287817, 0.7657672453742986, 1.0045876840265406, 0.9858056114400382, 0.8710937611903306, 0.8319330217638002, 1.0050985706445066, 0.9363425314153265, 0.9111662095698898, 0.8822483642571924, 1.0547921529408322, 0.9937743096656301, 0.9653849347701057, 0.9236053864198286, 1.0608213360421703, 1.0216834776263237, 0.9948854733452297, 0.9444045194208119, 1.0902032759381237, 1.070297753084291, 1.0352646449837608, 1.0035839360241054, 1.109605122267389, 1.0988987221493531, 1.0612028824132491, 1.0392747076439328, 1.1393068813062424, 1.1142247321092786, 1.0935193303224786, 1.0883457940193757 ], "peak_layer": 29, "peak_normalized": 1.1393068813062424, "population_cv": 0.09089893356811851, "rest_mean": 9.776359500527313e-05, "spike_contrast": 1.0740814107080978, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010500606003915891, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.00010625938011799008, 9.759956446941942e-05, 8.322180656250566e-05, 7.573072798550129e-05, 9.934893023455516e-05, 9.749147284310311e-05, 8.614701800979674e-05, 8.227420767070726e-05, 9.939945448422804e-05, 9.259981015929952e-05, 9.010999201564118e-05, 8.725015504751354e-05, 0.00010431391274323687, 9.827953908825293e-05, 9.547196532366797e-05, 9.134016727330163e-05, 0.00010491017019376159, 0.00010103962267749012, 9.838942787609994e-05, 9.339710231870413e-05, 0.0001078159039025195, 0.00010584734263829887, 0.00010238273534923792, 9.924966434482485e-05, 0.00010973465396091342, 0.00010867584205698222, 0.00010494790330994874, 0.00010277931141899899, 0.00011267201625742018, 0.000110191511339508, 0.00010814384586410597, 0.0001076322078006342 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.08070781361311674, "first_to_last_ratio": 0.8419197775247833, "imbalance_abs_log_ratio": 0.1720705453776983, "last_quartile_mean": 0.09586164355278015, "mean": 0.0877012787386775, "normalized": [ 1.074464079725317, 0.986898606287062, 0.8415149217948403, 0.7657672391640048, 1.0045876578450608, 0.9858055872189405, 0.8710938052118494, 0.8319330371629138, 1.0050985716236776, 0.9363425240221704, 0.9111662153888755, 0.8822483595926541, 1.0547922001671663, 0.993774279010469, 0.9653849224854107, 0.9236053602662656, 1.060821305580311, 1.021683475130349, 0.9948854782496288, 0.9444045803369281, 1.0902032654624292, 1.07029776221103, 1.0352646575744078, 1.0035839255047951, 1.109605076031185, 1.0988987393251362, 1.0612029192639736, 1.0392747445534645, 1.1393068879878778, 1.1142247084644445, 1.0935193626414328, 1.0883457447159286 ], "peak_layer": 29, "peak_normalized": 1.1393068879878778, "population_cv": 0.09089892559784572, "rest_mean": 0.08669773351263117, "spike_contrast": 1.0740813996684329, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0931204229593277, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.09423187375068665, 0.08655226975679398, 0.0738019347190857, 0.0671587660908699, 0.08810362219810486, 0.08645641058683395, 0.07639604061841965, 0.07296159118413925, 0.08814842998981476, 0.0821184366941452, 0.07991044223308563, 0.07737430930137634, 0.09250662475824356, 0.08715527504682541, 0.08466549217700958, 0.08100137114524841, 0.09303538501262665, 0.08960294723510742, 0.08725272864103317, 0.08282548934221268, 0.09561222046613693, 0.09386648237705231, 0.09079403430223465, 0.0880155935883522, 0.0973137840628624, 0.09637482464313507, 0.09306885302066803, 0.09114572405815125, 0.0999186709523201, 0.09771893173456192, 0.09590304642915726, 0.09544931352138519 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0474835562490625e-05, "first_to_last_ratio": 0.595870196340598, "imbalance_abs_log_ratio": 0.5177324270116084, "last_quartile_mean": 1.7579056020622374e-05, "mean": 1.45315484587627e-05, "normalized": [ 0.6891108502390517, 0.67218760206668, 0.6248497232673742, 0.6113105238885437, 0.8536003812586072, 0.8434482348779837, 0.7260190066641083, 0.7461464268756399, 0.9515026546685825, 0.8968707547028398, 0.8666952712848448, 0.9173542983424958, 1.0474470628626384, 1.0570493146141757, 0.9856965747654381, 1.0403130781822671, 1.05710126231986, 1.0899464822844398, 1.0639055355381295, 0.9944043245776518, 1.1636416255463544, 1.1162669452398895, 1.1491653646620976, 1.158233556446643, 1.241320970071456, 1.2666472916328237, 1.1327368714456698, 1.1012802199127885, 1.2948691657915095, 1.2555094531837419, 1.2151558495322814, 1.1702133232533922 ], "peak_layer": 29, "peak_normalized": 1.2948691657915095, "population_cv": 0.19478066534993646, "rest_mean": 1.408557606434139e-05, "spike_contrast": 1.202634475953173, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6939799388637765e-05, "top_five_layers": [ 29, 26, 30, 25, 31 ], "top_five_spike_overlap": 1 }, "values": [ 1.0013847713707946e-05, 9.767926712811459e-06, 9.080034033104312e-06, 8.883288501237985e-06, 1.2404135304677766e-05, 1.2256608897587284e-05, 1.0550180377322249e-05, 1.0842662959476002e-05, 1.3826806934957858e-05, 1.3032920833211392e-05, 1.2594424333656207e-05, 1.3330578440218233e-05, 1.522102775197709e-05, 1.5360563338617794e-05, 1.4323697541840374e-05, 1.5117359907890204e-05, 1.5361318219220266e-05, 1.5838610124774277e-05, 1.546019484521821e-05, 1.445023463020334e-05, 1.690951467026025e-05, 1.622108720766846e-05, 1.669915218371898e-05, 1.6830927052069455e-05, 1.8038315829471685e-05, 1.8406346498522907e-05, 1.6460420738440007e-05, 1.600330688233953e-05, 1.8816454030456953e-05, 1.8244496459374204e-05, 1.7658096112427302e-05, 1.7005011613946408e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.836043712610262e-05, "first_to_last_ratio": 0.6391636816233681, "imbalance_abs_log_ratio": 0.44759470463089907, "last_quartile_mean": 0.00010695294349716278, "mean": 9.121693358338234e-05, "normalized": [ 0.7254188545798665, 0.6902023390007371, 0.6496225556724757, 0.6277900804246503, 0.8680656002782756, 0.8588952084862168, 0.7971788633739051, 0.7782428701770085, 0.9734232055545582, 0.9237343700472408, 0.9096320021673064, 0.8918620199754489, 1.0844263684244682, 1.0342037426828012, 1.0147200800673386, 0.9791745321037664, 1.1290928536156495, 1.0928814238713915, 1.0660995437899934, 1.013549283136708, 1.170635644938427, 1.1503062295054498, 1.1129730585678779, 1.0777737723208793, 1.1919439416858, 1.1807148042349604, 1.1393605783805036, 1.1153320412921235, 1.222157091120308, 1.1946826876057626, 1.1714041050576325, 1.1645002478604691 ], "peak_layer": 29, "peak_normalized": 1.222157091120308, "population_cv": 0.17156610618115575, "rest_mean": 8.883977756744975e-05, "spike_contrast": 1.1712498490939807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010405357606941834, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 6.617048347834498e-05, 6.295814091572538e-05, 5.925657751504332e-05, 5.7265086070401594e-05, 7.91822822066024e-05, 7.834578718757257e-05, 7.271621143445373e-05, 7.098892820067704e-05, 8.879267988959327e-05, 8.426021668128669e-05, 8.297384192701429e-05, 8.135291864164174e-05, 9.891804802464321e-05, 9.433689410798252e-05, 9.255965414922684e-05, 8.931729826144874e-05, 0.00010299238783773035, 9.968929225578904e-05, 9.724633127916604e-05, 9.245285764336586e-05, 0.00010678179387468845, 0.00010492740693734959, 0.00010152198956348002, 9.831121860770509e-05, 0.00010872547136386856, 0.00010770118387881666, 0.0001039289782056585, 0.00010173716873396188, 0.00011148142220918089, 0.00010897529136855155, 0.00010685189045034349, 0.00010622214176692069 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.421776268121903e-05, "first_to_last_ratio": 0.5089233562442377, "imbalance_abs_log_ratio": 0.6754578508927926, "last_quartile_mean": 0.00010653423942130757, "mean": 8.540443025140121e-05, "normalized": [ 0.5542697148172375, 0.5395320644633284, 0.526750599523528, 0.5251350203104778, 0.7537064875235374, 0.7453106043033457, 0.7183783488935762, 0.7156010196903468, 0.915261682615852, 0.8828592813937833, 0.8733821992497209, 0.8643716402564832, 1.0724322757212523, 1.0352702478756863, 1.026729960575756, 1.0038080492636434, 1.1695135624376918, 1.1411903421843295, 1.1203495535033974, 1.0697851231751558, 1.23599072599408, 1.2140634563521424, 1.1785251437606976, 1.1385123493568967, 1.2633504856351931, 1.2489710766708417, 1.2127527443953423, 1.1862530196716095, 1.298334269521613, 1.274402551981099, 1.2503128844914202, 1.2448935143909348 ], "peak_layer": 29, "peak_normalized": 1.298334269521613, "population_cv": 0.24541141609846925, "rest_mean": 8.214501078193145e-05, "spike_contrast": 1.2539446328637511, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010300529538653791, "top_five_layers": [ 29, 30, 25, 31, 26 ], "top_five_spike_overlap": 1 }, "values": [ 4.73370891995728e-05, 4.6078428567852825e-05, 4.498683483689092e-05, 4.4848857214674354e-05, 6.436987314373255e-05, 6.365282752085477e-05, 6.135269359219819e-05, 6.111549737397581e-05, 7.816740253474563e-05, 7.540009391959757e-05, 7.459070911863819e-05, 7.382116746157408e-05, 9.159046749118716e-05, 8.841666567604989e-05, 8.768728730501607e-05, 8.572965452913195e-05, 9.98816394712776e-05, 9.746271098265424e-05, 9.56828152993694e-05, 9.136438893619925e-05, 0.00010555908374954015, 0.00010368639777880162, 0.00010065126843983307, 9.723399853101e-05, 0.0001078957284335047, 0.00010666766320355237, 0.00010357445717090741, 0.00010131126327905804, 0.00011088349856436253, 0.00010883962386287749, 0.00010678225953597575, 0.00010631942132022232 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014064586594031425, "first_to_last_ratio": 1.0563608173139223, "imbalance_abs_log_ratio": 0.054829809986132017, "last_quartile_mean": 0.00013314188072399702, "mean": 0.0001347423383322166, "normalized": [ 1.1217990599984589, 1.0213028410040081, 0.9566177087867302, 0.90173442925003, 1.162761655685041, 1.2034553361266669, 1.0271109783027095, 0.9557256445604886, 1.16094674777681, 1.05596774404836, 1.0011734009390578, 0.9818696062747876, 1.1123781705815696, 1.0319357929888284, 0.9826152250905227, 0.9109733958837348, 1.0154471845455557, 0.955267408663159, 0.9085227032176484, 0.853750823711138, 0.9825495622515935, 0.9600374683174735, 0.9243229318330685, 0.9067574744309754, 0.9933509912573504, 0.986398830187635, 0.9577583846169437, 0.9375602577665346, 1.0292265528944584, 1.007559112024919, 0.9944595916547317, 0.9986629853290112 ], "peak_layer": 6, "peak_normalized": 1.2034553361266669, "population_cv": 0.07980954026492859, "rest_mean": 0.0001359050228428613, "spike_contrast": 0.9452471975467027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012846384197473526, "top_five_layers": [ 6, 5, 9, 1, 13 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001511538284830749, 0.0001376127329422161, 0.00012889690697193146, 0.00012150180555181578, 0.00015667322441004217, 0.0001621563860680908, 0.00013839533494319767, 0.00012877670815214515, 0.00015642867947462946, 0.0001422835630364716, 0.00013490044511854649, 0.00013229940668679774, 0.000149884435813874, 0.00013904544175602496, 0.00013239987310953438, 0.0001227466855198145, 0.00013682372809853405, 0.0001287149643758312, 0.0001224164734594524, 0.00011503638233989477, 0.00013239102554507554, 0.0001293576933676377, 0.0001245454332092777, 0.00012217862240504473, 0.00013384643534664065, 0.000132909684907645, 0.00012905060430057347, 0.00012632906145881861, 0.00013868039241060615, 0.00013576087076216936, 0.00013399581075645983, 0.0001345621858490631 ] } } } } }, "learned": { "bits_per_byte": 8.038491115106986, "forward": { "activation_sha256": { "attention_branch_output": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", "mlp_branch_output": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", "post_attention_state": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", "post_mlp_state": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", "pre_attention_input": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", "pre_mlp_input": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd" }, "logits_sha256": "5e5a50bae6f74c41645e35a857f0cba662c8b3f5ba9a8a4635e860fdaa3b0c88", "loss_nats": 5.571857452392578, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.571857452392578, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0023513585329055786, 0.0023208872880786657, 0.002778682392090559, 0.002497476991266012, 0.002628475893288851, 0.002920303260907531, 0.002932395786046982, 0.0032228652853518724, 0.003561305580660701, 0.003354643704369664, 0.0034162327647209167, 0.0036349669098854065, 0.0035210708156228065, 0.0038089773152023554, 0.003915763925760984, 0.0037244565319269896, 0.004162859171628952, 0.004167158622294664, 0.004440344404429197, 0.0041700867004692554, 0.004737259820103645, 0.004552462603896856, 0.004706811159849167, 0.005186605267226696, 0.005580419674515724, 0.004905648995190859, 0.005227713380008936, 0.005349088925868273, 0.005235649179667234, 0.004925078712403774, 0.005810525268316269, 0.005564783234149218 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024914435198297724, "first_to_last_ratio": 5.005386356076312, "imbalance_abs_log_ratio": 1.61051460380912, "last_quartile_mean": 4.977524895366514e-05, "mean": 0.00012233095264946314, "normalized": [ 2.946204230178919, 2.5883907308658176, 2.285503889899197, 2.0157125547168477, 1.8168856694000293, 1.687577213316311, 1.555459627812154, 1.3974018168018878, 1.2756749767931916, 1.1684180974434406, 1.0925035606597346, 1.0000212864934148, 0.922022236502042, 0.863306019148705, 0.7954975128821132, 0.747136294956448, 0.6955650087701328, 0.6567702950630189, 0.6067855157189815, 0.5734103460233209, 0.549704279492861, 0.527795032775898, 0.5009080420334019, 0.4762252632696638, 0.45754883501456406, 0.43797117166433, 0.4282553488048722, 0.41570642703193256, 0.40058854659018955, 0.39033614610916745, 0.3703926654711839, 0.3543213582962283 ], "peak_layer": 1, "peak_normalized": 2.946204230178919, "population_cv": 0.6883820030702144, "rest_mean": 0.00013360270127628205, "spike_contrast": 0.46004691130861425, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.146351006464101e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036041197017766535, 0.0003166403039358556, 0.00027958786813542247, 0.0002465840370859951, 0.00022226135479286313, 0.0002064429281745106, 0.00019028085807804018, 0.0001709454954834655, 0.00015605453518219292, 0.00014293369895312935, 0.00013364700134843588, 0.00012233355664648116, 0.0001127918585552834, 0.00010560904775047675, 9.731396858114749e-05, 9.139789472101256e-05, 8.508913015248254e-05, 8.03433358669281e-05, 7.42286501917988e-05, 7.014583388809115e-05, 6.724584818584844e-05, 6.456566916313022e-05, 6.127655797172338e-05, 5.825709013151936e-05, 5.597238487098366e-05, 5.3577430662699044e-05, 5.2388884796528146e-05, 5.085376324132085e-05, 4.900437852484174e-05, 4.775019260705449e-05, 4.5310487621463835e-05, 4.334446930442937e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002986813742609229, "first_to_last_ratio": 2.3520308544062645, "imbalance_abs_log_ratio": 0.8552791483203458, "last_quartile_mean": 0.0001269887143280357, "mean": 0.00018719729814620223, "normalized": [ 2.1746021240914017, 1.94336263487061, 1.7490368812517914, 1.5797937988337565, 1.4558020735282082, 1.372777068369229, 1.290232936323773, 1.1987380034727069, 1.1282687150208124, 1.0677045109732362, 1.0238712087163733, 0.9743741602296072, 0.9332108464677616, 0.9012687760530631, 0.8680188022864366, 0.8423083374767277, 0.8168968442148058, 0.7977443195284458, 0.774889242886524, 0.759928616461841, 0.7477907292059667, 0.7367409087088999, 0.7235787757136496, 0.7121132246295925, 0.7033660136533255, 0.6946945949961153, 0.6884818791691059, 0.6826464924874314, 0.674695217464828, 0.6690870522688999, 0.6604539573176332, 0.6535212533274407 ], "peak_layer": 1, "peak_normalized": 2.1746021240914017, "population_cv": 0.3960989175249597, "rest_mean": 0.00019674027215741161, "spike_contrast": 0.68956516628749, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013566523848567157, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004070796421729028, 0.0003637922345660627, 0.0003274149785283953, 0.0002957331307698041, 0.0002725222148001194, 0.000256980158155784, 0.0002415281196590513, 0.00022440051543526351, 0.00021120885503478348, 0.00019987139967270195, 0.00019166592392139137, 0.0001824002101784572, 0.00017469454905949533, 0.00016871507978066802, 0.00016249077452812344, 0.00015767784498166293, 0.00015292088210117072, 0.0001493355812272057, 0.00014505717263091356, 0.0001422565837856382, 0.00013998440408613533, 0.0001379159075440839, 0.00013545199180953205, 0.0001333056716248393, 0.00013166821736376733, 0.000130044951220043, 0.0001288819476030767, 0.00012778957898262888, 0.00012630112178158015, 0.00012525128840934485, 0.00012363519635982811, 0.00012233741290401667 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.264873631298542, "first_to_last_ratio": 2.3520309299879743, "imbalance_abs_log_ratio": 0.8552791804550044, "last_quartile_mean": 0.11261485889554024, "mean": 0.1660084358882159, "normalized": [ 2.1746021175238424, 1.943362568695452, 1.7490370652499811, 1.5797937888156908, 1.4558020518807187, 1.3727770906872316, 1.2902329816871188, 1.1987379486864336, 1.128268732118104, 1.067704514212167, 1.0238712059346726, 0.9743741284397964, 0.9332107750795899, 0.9012687970282692, 0.8680188148399602, 0.8423083473020997, 0.8168968752199287, 0.7977442908994873, 0.7748892254753884, 0.7599285915530729, 0.7477907763310051, 0.7367409150028685, 0.723578786768912, 0.7121132849551643, 0.7033659848486875, 0.694694622945607, 0.6884818276985868, 0.6826464794318051, 0.6746951840389442, 0.6690870220520635, 0.6604539434158253, 0.6535212611815249 ], "peak_layer": 1, "peak_normalized": 2.1746021175238424, "population_cv": 0.3960989241805914, "rest_mean": 0.17447124007675383, "spike_contrast": 0.6895651868880185, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12030929327011108, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.3610022962093353, 0.3226145803928375, 0.2903549075126648, 0.2622590959072113, 0.24167542159557343, 0.22789257764816284, 0.2141895592212677, 0.19900061190128326, 0.1873021274805069, 0.17724795639514923, 0.1699712574481964, 0.1617543250322342, 0.15492086112499237, 0.1496182233095169, 0.1440984457731247, 0.13983029127120972, 0.13561177253723145, 0.13243228197097778, 0.1286381483078003, 0.1261545568704605, 0.12413957715034485, 0.12230520695447922, 0.12012018263339996, 0.11821681261062622, 0.11676468700170517, 0.11532516777515411, 0.11429379135370255, 0.1133250743150711, 0.1120050922036171, 0.11107409000396729, 0.1096409261226654, 0.10849004238843918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010120303977600997, "first_to_last_ratio": 2.7021675066537045, "imbalance_abs_log_ratio": 0.994054231196686, "last_quartile_mean": 3.7452541164384456e-05, "mean": 6.280763545873924e-05, "normalized": [ 1.9944816746100333, 1.8472480968114133, 1.7373223217437128, 1.6248488809553971, 1.5471371990089653, 1.475823179261819, 1.3807460832170038, 1.2829322622274897, 1.184046410564159, 1.1043973349513228, 1.0901910170788929, 1.0456150919711364, 0.9888694060739084, 0.9664075020831666, 0.9198699716804313, 0.9041232034205934, 0.8692950074065211, 0.8446819175937094, 0.7923092109368705, 0.7644112750281646, 0.7503465778061283, 0.730017034806627, 0.7090250315296005, 0.6754099655367306, 0.6572034579968898, 0.6267819521510305, 0.6185439744898635, 0.6068927943478243, 0.594580659914726, 0.5858446061910916, 0.5555293303136664, 0.5250675682911103 ], "peak_layer": 1, "peak_normalized": 1.9944816746100333, "population_cv": 0.40483316550850257, "rest_mean": 6.624576713807915e-05, "spike_contrast": 0.6678422833882899, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4241724390303715e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012526867794804275, 0.0001160212850663811, 0.0001091171070584096, 0.00010205291619058698, 9.717202920001e-05, 9.269296424463391e-05, 8.67213966557756e-05, 8.057794184423983e-05, 7.43671553209424e-05, 6.936458521522582e-05, 6.847231998108327e-05, 6.567261152667925e-05, 6.210854917299002e-05, 6.0697770095430315e-05, 5.777485785074532e-05, 5.6785840570228174e-05, 5.4598363931290805e-05, 5.305247395881452e-05, 4.9763068091124296e-05, 4.801086470251903e-05, 4.712749432655983e-05, 4.585064380080439e-05, 4.453218571143225e-05, 4.242090290063061e-05, 4.1277395212091506e-05, 3.936669236281887e-05, 3.8849284464959055e-05, 3.811750138993375e-05, 3.734420533874072e-05, 3.679551446111873e-05, 3.4891483664978296e-05, 3.297825242043473e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021292950441420544, "first_to_last_ratio": 1.6819067487661699, "imbalance_abs_log_ratio": 0.5199281193278776, "last_quartile_mean": 0.0001266000654140953, "mean": 0.0001612693722563563, "normalized": [ 1.5193752902631465, 1.4431842082185322, 1.3854135927323608, 1.3303138916593422, 1.2819858609596173, 1.2449680699964556, 1.2025089225516594, 1.1549255897117912, 1.1161361523436344, 1.0820631347545688, 1.0617857510438193, 1.0341003696475617, 1.0077971864601107, 0.9864740856164548, 0.9634556759892069, 0.9447446572547388, 0.9240188125766456, 0.9078710595764491, 0.88634376010722, 0.8724890232241534, 0.8607832896405347, 0.849628883322604, 0.8359853832855236, 0.8234683594489096, 0.8139294954165563, 0.8040629029670215, 0.7969416676465471, 0.7902390259734583, 0.7809695092222964, 0.7743360767166025, 0.7639958481267083, 0.7557044635457686 ], "peak_layer": 1, "peak_normalized": 1.5193752902631465, "population_cv": 0.21463654613558777, "rest_mean": 0.00016614451305940747, "spike_contrast": 0.8122062500591205, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013494361191987992, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024502869928255677, 0.00023274141130968928, 0.00022342478041537106, 0.0002145388862118125, 0.00020674505503848195, 0.00020077521912753582, 0.00019392785907257348, 0.00018625412485562265, 0.00017999857664108276, 0.00017450364248361439, 0.00017123352154158056, 0.00016676871746312827, 0.0001625268196221441, 0.00015908805653452873, 0.0001553758920636028, 0.0001523583778180182, 0.0001490159338572994, 0.000146411795867607, 0.00014294010179582983, 0.0001407057570759207, 0.00013881798076909035, 0.00013701911666430533, 0.0001348188379779458, 0.0001328002254012972, 0.00013126189878676087, 0.0001296707196161151, 0.00012852228246629238, 0.00012744135165121406, 0.0001259464625036344, 0.00012487669300753623, 0.00012320913083385676, 0.00012187198444735259 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016320452232321259, "first_to_last_ratio": 1.2889611434881496, "imbalance_abs_log_ratio": 0.25383657880769345, "last_quartile_mean": 0.0001266171002498595, "mean": 0.0001437355554116948, "normalized": [ 1.188749445631, 1.1713423911172207, 1.1549430883999143, 1.1417765084687417, 1.1273283201397843, 1.1145504039766105, 1.0995991496765185, 1.0853099095412984, 1.0687879028680622, 1.0554743211679045, 1.0473286818228558, 1.0363809984280803, 1.0249947395076546, 1.015899258153915, 1.005968438235508, 0.9970397006238613, 0.9860146691908593, 0.977410712347053, 0.964005406402611, 0.9549550784847994, 0.9474810711889571, 0.940343993567483, 0.9294782140508817, 0.9176125795270506, 0.909513915953909, 0.8990879268536105, 0.8931112722873075, 0.8859642730588357, 0.8770245001904262, 0.8704731011044752, 0.8603223859779169, 0.851727642054894 ], "peak_layer": 1, "peak_normalized": 1.188749445631, "population_cv": 0.0978236443065259, "rest_mean": 0.0001456284474072702, "spike_contrast": 0.9168122095142401, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001335139386355877, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017086556181311607, 0.00016836354916449636, 0.00016600638628005981, 0.00016411388060078025, 0.0001620371622266248, 0.00016020052134990692, 0.0001580514945089817, 0.00015599762264173478, 0.00015362282283604145, 0.0001517091877758503, 0.00015053836978040636, 0.0001489647984271869, 0.00014732818817719817, 0.0001460208441130817, 0.00014459343219641596, 0.0001433100551366806, 0.00014172536612022668, 0.00014048867160454392, 0.00013856185250915587, 0.00013726099859923124, 0.0001361867180094123, 0.00013516086619347334, 0.00013359906733967364, 0.0001318935537710786, 0.0001307294878643006, 0.00012923090253025293, 0.00012837184476666152, 0.0001273445668630302, 0.00012605960364453495, 0.0001251179346581921, 0.00012365891598165035, 0.00012242354569025338 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0004412600719660986, "first_to_last_ratio": 3.098863205201153, "imbalance_abs_log_ratio": 1.131035336237652, "last_quartile_mean": 0.00014239417578210123, "mean": 0.0002548141455918085, "normalized": [ 2.1785577639630773, 1.9956115105027457, 1.9014578504944815, 1.7829105267225334, 1.6475854563109211, 1.565091756282257, 1.45799103716316, 1.3243443696030393, 1.2370613678820481, 1.1609137324491365, 1.1300355744023234, 1.0677196003316773, 1.0048909110245325, 0.948494062464312, 0.8853730946340664, 0.8440976906413151, 0.7906677717627395, 0.747069444517614, 0.6991826523616375, 0.6712814178889162, 0.6509454457824981, 0.6314362826751919, 0.6113540983289655, 0.5954000197507868, 0.5831757760496624, 0.5733601746441271, 0.5653129211221045, 0.559900286080882, 0.5532114024202266, 0.5496528914136888, 0.545014697375157, 0.5408984129541747 ], "peak_layer": 1, "peak_normalized": 2.1785577639630773, "population_cv": 0.4822008017525282, "rest_mean": 0.00027300682214209465, "spike_contrast": 0.573515676244785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015657369222026317, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005551273352466524, 0.0005085100419819355, 0.000484518357552588, 0.00045431082253344357, 0.00041982808033935726, 0.0003988075186498463, 0.0003715167404152453, 0.0003374616790097207, 0.000315220735501498, 0.0002958172408398241, 0.0002879490493796766, 0.0002720700576901436, 0.0002560604189056903, 0.00024168970412574708, 0.00022560558863915503, 0.00021508803183678538, 0.0002014733327087015, 0.0001903638622025028, 0.0001781616301741451, 0.0001710520009510219, 0.00016587010759394616, 0.0001608988968655467, 0.00015578167221974581, 0.00015171634731814265, 0.00014860143710393459, 0.00014610028301831335, 0.0001440497289877385, 0.0001426705130143091, 0.00014096609083935618, 0.0001400593318976462, 0.0001388774544466287, 0.00013782856694888324 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027412420604377985, 0.002683233004063368, 0.002711885841563344, 0.0027249811682850122, 0.0027012599166482687, 0.002700963756069541, 0.0026683746837079525, 0.002738022478297353, 0.0027370047755539417, 0.002654933836311102, 0.002663377905264497, 0.002684312639757991, 0.0027356327045708895, 0.002652956871315837, 0.002679797587916255, 0.002698024269193411, 0.0026484597474336624, 0.0026680538430809975, 0.002654345240443945, 0.002591830212622881, 0.002620960585772991, 0.002651234855875373, 0.0026270023081451654, 0.002615687670186162, 0.00258000404573977, 0.002626793459057808, 0.002573782578110695, 0.002592637902125716, 0.002581534907221794, 0.002551236655563116, 0.002563327783718705, 0.002589041367173195 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024784023116808385, "first_to_last_ratio": 4.996872150813687, "imbalance_abs_log_ratio": 1.6088121468463858, "last_quartile_mean": 4.959907391821616e-05, "mean": 0.00012175017718618619, "normalized": [ 2.94187592163941, 2.5796687009574604, 2.293010133276413, 2.014581184592976, 1.8111575385173901, 1.6842185280489494, 1.5559062012549134, 1.4047478191310576, 1.2749623745204015, 1.1699150302730006, 1.0919472369387608, 1.0017565099771182, 0.923449750141626, 0.8652102016053241, 0.794546741948017, 0.743905553038253, 0.6942834150416333, 0.6562793060663384, 0.6082995898528493, 0.5744438165059796, 0.5512557468449423, 0.5265581646918154, 0.5020472770644135, 0.47690127545745264, 0.4572061982209032, 0.43893553328289897, 0.42961039869683454, 0.41603121061550513, 0.3999104909112459, 0.3911954601706125, 0.3717234123738856, 0.35445927834161867 ], "peak_layer": 1, "peak_normalized": 2.94187592163941, "population_cv": 0.6874421460933116, "rest_mean": 0.0001329603533290706, "spike_contrast": 0.46040210094136547, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.121522601461038e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000358173914719373, 0.0003140751214232296, 0.00027917439001612365, 0.0002452756161801517, 0.00022050875122658908, 0.0002050539042102173, 0.00018943185568787158, 0.0001710282958811149, 0.00015522689500357956, 0.00014243736222852021, 0.00013294476957526058, 0.00012196403258712962, 0.00011243017070228234, 0.0001053394953487441, 9.673620661487803e-05, 9.057063289219514e-05, 8.45291287987493e-05, 7.990212179720402e-05, 7.406058284686878e-05, 6.993863644311205e-05, 6.711548485327512e-05, 6.410854985006154e-05, 6.112434493843466e-05, 5.806281478726305e-05, 5.566493564401753e-05, 5.3440478950506076e-05, 5.23051421623677e-05, 5.0651873607421294e-05, 4.8689173127058893e-05, 4.762811659020372e-05, 4.525739132077433e-05, 4.315547994337976e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002960815872938838, "first_to_last_ratio": 2.3434901925615312, "imbalance_abs_log_ratio": 0.8516413534183918, "last_quartile_mean": 0.00012634214908757713, "mean": 0.00018583460405352525, "normalized": [ 2.1691817718266027, 1.9356770461635995, 1.7512361881653136, 1.5771501673188, 1.4517248962140685, 1.3691574500653627, 1.2898282592750647, 1.2020702391870204, 1.1274711846152252, 1.06796916249687, 1.023464646242978, 0.9751293351351744, 0.9332415616222968, 0.9026335726010739, 0.8668647658197504, 0.8407791679025106, 0.8171800119835181, 0.7975248027052753, 0.7770671945621679, 0.7612080123915693, 0.7492445405290613, 0.7365475788483447, 0.7249228575947372, 0.7138182430319103, 0.7048361054118131, 0.6962681268232633, 0.6911654119184109, 0.6833348384348117, 0.6760742523950837, 0.6702828385523486, 0.6624247014500871, 0.654521068715886 ], "peak_layer": 1, "peak_normalized": 2.1691817718266027, "population_cv": 0.3943796352106793, "rest_mean": 0.00019526833030222742, "spike_contrast": 0.6908057343541223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013489248231053353, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004031090356875211, 0.0003597157774493098, 0.0003254402836319059, 0.0002930890768766403, 0.00026978072128258646, 0.00025443683261983097, 0.00023969472385942936, 0.00022338624694384634, 0.00020952316117472947, 0.0001984656264539808, 0.0001901951472973451, 0.00018121277389582247, 0.0001734285760903731, 0.00016774055256973952, 0.0001610934705240652, 0.0001562458637636155, 0.00015186032396741211, 0.00014820770593360066, 0.00014440597442444414, 0.00014145878958515823, 0.00013923556252848357, 0.0001368760276818648, 0.00013471575221046805, 0.00013265213056001812, 0.00013098293857183307, 0.00012939071166329086, 0.00012844245065934956, 0.00012698725913651288, 0.00012563799100462347, 0.0001245617459062487, 0.00012310143210925162, 0.00012163266364950687 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.26256811060011387, "first_to_last_ratio": 2.3434901455089974, "imbalance_abs_log_ratio": 0.8516413333404181, "last_quartile_mean": 0.11204148270189762, "mean": 0.16479998547583818, "normalized": [ 2.1691817568110046, 1.935676923537455, 1.7512361570514487, 1.5771502517930829, 1.4517248703109182, 1.369157420355369, 1.2898282606548555, 1.2020702692325518, 1.1274712340116186, 1.0679692103147351, 1.0234646460792782, 0.975129361875517, 0.9332415430463463, 0.9026335787242323, 0.8668647305888977, 0.8407791967519535, 0.8171800231493583, 0.7975247749486774, 0.7770672329681078, 0.7612079835803747, 0.7492445111174014, 0.7365475542475244, 0.7249228843083022, 0.7138182179223472, 0.7048361082067401, 0.6962681206044983, 0.6911654218524023, 0.6833348067031066, 0.676074287607254, 0.6702828621875807, 0.6624247142471237, 0.6545210852099367 ], "peak_layer": 1, "peak_normalized": 2.1691817568110046, "population_cv": 0.3943796257753467, "rest_mean": 0.17316590856622766, "spike_contrast": 0.6908057237027376, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11962400078773498, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.35748112201690674, 0.3189995288848877, 0.28860369324684143, 0.2599143385887146, 0.2392442375421524, 0.22563712298870087, 0.2125636786222458, 0.19810116291046143, 0.1858072429895401, 0.17600131034851074, 0.16866695880889893, 0.16070130467414856, 0.1537981927394867, 0.14875400066375732, 0.1428592950105667, 0.13856039941310883, 0.13467125594615936, 0.13143207132816315, 0.12806066870689392, 0.12544706463813782, 0.12347548454999924, 0.12138302624225616, 0.11946728080511093, 0.11763723194599152, 0.11615698039531708, 0.11474497616291046, 0.11390405148267746, 0.11261356621980667, 0.11141703277826309, 0.11046260595321655, 0.10916758328676224, 0.10786506533622742 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.0001010398200378404, "first_to_last_ratio": 2.701478580108855, "imbalance_abs_log_ratio": 0.9937992453824708, "last_quartile_mean": 3.740167358046165e-05, "mean": 6.270599271829269e-05, "normalized": [ 1.9907220064146427, 1.8444741506775382, 1.742884685905627, 1.6239253294262157, 1.5436270879110234, 1.4787029723792133, 1.3841644782908762, 1.2821103017907618, 1.1824374683113494, 1.1042100658007115, 1.0908679078034196, 1.0498354286471192, 0.9920033594422748, 0.9660018389944104, 0.9202838356691797, 0.8989347700163286, 0.8686518679586676, 0.8410583182307008, 0.7938538755561205, 0.7653997098760733, 0.7511903220519177, 0.7265360595007238, 0.7106247592487483, 0.6758121071782397, 0.6556340454607258, 0.6278169583683687, 0.6205082786340235, 0.6073860029650574, 0.5911469097961847, 0.5867782134359579, 0.5577710930141501, 0.5246457912436486 ], "peak_layer": 1, "peak_normalized": 1.9907220064146427, "population_cv": 0.40468494452628273, "rest_mean": 6.614368087159366e-05, "spike_contrast": 0.6673725457789733, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4142476690467446e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001248301996383816, 0.00011565958266146481, 0.0001092893144232221, 0.00010182984988205135, 9.679466893430799e-05, 9.272353781852871e-05, 8.679540769662708e-05, 8.039599924813956e-05, 7.414591527776793e-05, 6.92405883455649e-05, 6.84039550833404e-05, 6.583097274415195e-05, 6.220455543370917e-05, 6.0574104281840846e-05, 5.770731149823405e-05, 5.636859714286402e-05, 5.446967770694755e-05, 5.273939677863382e-05, 4.9779395340010524e-05, 4.799514863407239e-05, 4.710413486463949e-05, 4.5558164856629446e-05, 4.456043097889051e-05, 4.237746907165274e-05, 4.111218368052505e-05, 3.936788561986759e-05, 3.890958760166541e-05, 3.808674227911979e-05, 3.7068453821120784e-05, 3.679451037896797e-05, 3.497559009701945e-05, 3.2898435165407136e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021103879953443538, "first_to_last_ratio": 1.6754332028061545, "imbalance_abs_log_ratio": 0.5160717603794217, "last_quartile_mean": 0.00012596073611348402, "mean": 0.0001601102544555033, "normalized": [ 1.5150372860566723, 1.439510593902249, 1.3841055212355642, 1.3272119947749268, 1.2794362361966427, 1.2428436489609036, 1.2018556832108258, 1.1546727794959446, 1.1147996317367452, 1.0814138049096294, 1.0614905005088058, 1.0346901524019803, 1.0074919017055521, 0.9875011592687566, 0.9623166883068708, 0.9431047552100263, 0.9243559814545742, 0.9073018837351503, 0.8887802361951885, 0.87382889642758, 0.8622859040817829, 0.849391060863955, 0.8374811583413132, 0.8253926631762389, 0.8155827011133436, 0.8058320881575772, 0.7999867009830005, 0.7910419816457831, 0.7825317922950641, 0.7756782882808636, 0.7662211489344684, 0.7568251764320209 ], "peak_layer": 1, "peak_normalized": 1.5150372860566723, "population_cv": 0.21304026877727222, "rest_mean": 0.00016491277050658095, "spike_contrast": 0.8136220583009953, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013417666777968407, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024257300538010895, 0.00023048040748108178, 0.0002216094871982932, 0.00021250025019980967, 0.0002048508613370359, 0.00019899201288353652, 0.0001924294192576781, 0.00018487495253793895, 0.00017849085270427167, 0.00017314543947577477, 0.00016995551413856447, 0.00016566450358368456, 0.00016130978474393487, 0.00015810906188562512, 0.00015407676983159035, 0.00015100074233487248, 0.00014799887139815837, 0.0001452683354727924, 0.00014230282977223396, 0.00013990896695759147, 0.00013806081551592797, 0.00013599621888715774, 0.00013408932136371732, 0.00013215382932685316, 0.00013058315380476415, 0.00012902198068331927, 0.00012808607425540686, 0.0001266539329662919, 0.00012529136438388377, 0.00012419404811225832, 0.00012267986312508583, 0.00012117547157686204 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016164849512279034, "first_to_last_ratio": 1.282986844747637, "imbalance_abs_log_ratio": 0.24919083207168544, "last_quartile_mean": 0.00012599388355738483, "mean": 0.00014267631695474847, "normalized": [ 1.1849632230478688, 1.1695181917643926, 1.1537907432295644, 1.136648454640294, 1.1262413432306055, 1.112705815659591, 1.0977204646042988, 1.082200264843305, 1.06813080649915, 1.0535985061590996, 1.046252801833764, 1.0365714691025223, 1.024670983408149, 1.0169605538784676, 1.0046472025165352, 0.995758555548206, 0.9858673222379531, 0.9760103584099877, 0.9653106304982666, 0.9567951741123486, 0.9498417329681972, 0.9401737612553472, 0.9304537733640251, 0.9205681496416466, 0.911480005330001, 0.9022600866995591, 0.8969925816929462, 0.8873573495747816, 0.8789659160771423, 0.8714788501373707, 0.8627683840784355, 0.8532965439561776 ], "peak_layer": 1, "peak_normalized": 1.1849632230478688, "population_cv": 0.09608600673281108, "rest_mean": 0.00014451252193086676, "spike_contrast": 0.918680321330362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013276081008370966, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016906618839129806, 0.00016686254821252078, 0.00016461861378047615, 0.00016217281518038362, 0.00016068796685431153, 0.00015875676763243973, 0.0001566187129355967, 0.00015440434799529612, 0.00015239696949720383, 0.0001503235544078052, 0.00014927549636922777, 0.00014789419947192073, 0.00014619628200307488, 0.0001450961863156408, 0.00014333936269395053, 0.00014207116328179836, 0.00014065991854295135, 0.00013925356324762106, 0.00013772696547675878, 0.0001365120115224272, 0.00013551992014981806, 0.00013414052955340594, 0.00013275371748022735, 0.0001313432730967179, 0.00013004661013837904, 0.00012873114610556513, 0.0001279795978916809, 0.00012660487846005708, 0.0001254076196346432, 0.00012433939264155924, 0.00012309661542531103, 0.00012174520816188306 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00043634913890855387, "first_to_last_ratio": 3.0823770886184128, "imbalance_abs_log_ratio": 1.125701081333854, "last_quartile_mean": 0.00014156254292174708, "mean": 0.0002526328457861382, "normalized": [ 2.161262991989335, 1.9867412804887739, 1.8904526133838582, 1.7788259407438674, 1.6404609794852394, 1.5723911781894129, 1.459249713339863, 1.3282687448916313, 1.2383315143134188, 1.1700416770361293, 1.131363959578303, 1.0685850444189446, 1.0069989316428685, 0.9451411723514572, 0.8866050347634621, 0.8407661253267313, 0.7895843736134068, 0.7479926194272515, 0.7047300318080335, 0.6729704299307414, 0.6547401608613147, 0.6315820650248338, 0.6127131155987415, 0.5974089218754778, 0.5839699651029305, 0.5735248801032955, 0.5690230706448541, 0.5599717580649735, 0.5552403508520525, 0.5513694456095706, 0.5470291494736481, 0.5426627600655782 ], "peak_layer": 1, "peak_normalized": 2.161262991989335, "population_cv": 0.4790543297146424, "rest_mean": 0.0002705939723118381, "spike_contrast": 0.5751893185853856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015564276254735886, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005460060201585293, 0.0005019161035306752, 0.0004775904235430062, 0.00044938985956832767, 0.0004144343256484717, 0.0003972376580350101, 0.000368654407793656, 0.00033556431299075484, 0.00031284321448765695, 0.000295590958558023, 0.0002858196967281401, 0.0002699596807360649, 0.0002544010058045387, 0.00023877370404079556, 0.0002239855530206114, 0.00021240513888187706, 0.00019947494729422033, 0.0001889675040729344, 0.0001780379534466192, 0.00017001443484332412, 0.00016540887008886784, 0.00015955837443470955, 0.00015479145804420114, 0.00015092511603143066, 0.00014752999413758516, 0.00014489122258964926, 0.00014375391765497625, 0.00014146725879982114, 0.00014027194993104786, 0.00013929403212387115, 0.00013819753075949848, 0.00013709443737752736 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0023513585329055786, 0.004408807028084993, 0.005783648230135441, 0.006751363631337881, 0.002628475893288851, 0.004723479505628347, 0.006024772301316261, 0.007445912808179855, 0.003561305580660701, 0.005543973762542009, 0.006855612155050039, 0.007876631803810596, 0.0035210708156228065, 0.0058924350887537, 0.007390674669295549, 0.008790239691734314, 0.004162859171628952, 0.006355749908834696, 0.007784162648022175, 0.00937180407345295, 0.004737259820103645, 0.006999304983764887, 0.008728929795324802, 0.010217384435236454, 0.005580419674515724, 0.008008211851119995, 0.010209310799837112, 0.012040933594107628, 0.005235649179667234, 0.007745372597128153, 0.010180563665926456, 0.011906592175364494 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024914481400628574, "first_to_last_ratio": 5.005367240493075, "imbalance_abs_log_ratio": 1.6105107847992817, "last_quartile_mean": 4.977553135177004e-05, "mean": 0.0001223312358433759, "normalized": [ 2.9462138255793877, 2.588435651553335, 2.2854731426364707, 2.0157214492628985, 1.8168708763555168, 1.6875649797632875, 1.5554268829788211, 1.3974214212146046, 1.275680469442624, 1.1684152736282016, 1.0925056707858531, 1.0000365768115411, 0.9220243844233116, 0.8633153808206192, 0.7954940059538663, 0.747131175135411, 0.6955592946052511, 0.6567700236831091, 0.6067846463225012, 0.5734077695641915, 0.5496986650820618, 0.5277889932643854, 0.50090777460584, 0.47622023530363405, 0.457550957846772, 0.43796965221258716, 0.42825768814458814, 0.4157111150450942, 0.4005890169587395, 0.39033503431894295, 0.37039430607646523, 0.3543236606200867 ], "peak_layer": 1, "peak_normalized": 2.9462138255793877, "population_cv": 0.6883836085455175, "rest_mean": 0.00013360307773971862, "spike_contrast": 0.46004396487681354, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.14632896031253e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036041397834196687, 0.00031664653215557337, 0.0002795847540255636, 0.0002465856960043311, 0.0002222600596724078, 0.0002064419095404446, 0.00019027729285880923, 0.00017094828945118934, 0.00015605556836817414, 0.00014293368440121412, 0.00013364756887312979, 0.00012233571032993495, 0.00011279238242423162, 0.00010561043745838106, 9.73137648543343e-05, 9.139747999142855e-05, 8.508862811140716e-05, 8.0343488662038e-05, 7.422871567541733e-05, 7.014568109298125e-05, 6.724531704094261e-05, 6.456507981056347e-05, 6.127666711108759e-05, 5.8256609918316826e-05, 5.597277413471602e-05, 5.357736881705932e-05, 5.238929225015454e-05, 5.0854454457294196e-05, 4.9004549509845674e-05, 4.775016714120284e-05, 4.531079321168363e-05, 4.334485129220411e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00029868123783671763, "first_to_last_ratio": 2.352020548935129, "imbalance_abs_log_ratio": 0.8552747667903616, "last_quartile_mean": 0.00012698921273113228, "mean": 0.0001871975605354237, "normalized": [ 2.1745956556501205, 1.943389916860048, 1.7490167059616495, 1.579801534641211, 1.4557872843351107, 1.3727714128797337, 1.290211227528517, 1.198748061313539, 1.1282731192017241, 1.0677025479901412, 1.0238711728275764, 0.9743826669000581, 0.9332092274716193, 0.901273964826617, 0.8680163418407979, 0.8423074677807364, 0.8168973316425937, 0.7977426572048737, 0.7748904888155362, 0.7599280177062464, 0.7477886704872306, 0.7367391764207782, 0.7235805599766263, 0.7121146362854748, 0.7033711688788506, 0.6946931548498627, 0.6884857337520279, 0.6826539310873735, 0.6746941940291191, 0.6690858034871798, 0.660454275348722, 0.6535218920183048 ], "peak_layer": 1, "peak_normalized": 2.1745956556501205, "population_cv": 0.39609817473730136, "rest_mean": 0.00019674051630621156, "spike_contrast": 0.6895661448911525, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013566559937316924, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004070790018886328, 0.00036379785160534084, 0.00032741166069172323, 0.00029573499341495335, 0.00027251982828602195, 0.00025697945966385305, 0.00024152439436875284, 0.000224402712774463, 0.00021120997553225607, 0.00019987131236121058, 0.00019166618585586548, 0.0001824020582716912, 0.00017469449085183442, 0.00016871628758963197, 0.00016249054169747978, 0.00015767790318932384, 0.00015292118769139051, 0.0001493354793637991, 0.0001450576091883704, 0.00014225667109712958, 0.00013998421491123736, 0.00013791577657684684, 0.00013545251567848027, 0.00013330612273421139, 0.00013166936696507037, 0.00013004486390855163, 0.00012888284982182086, 0.00012779115058947355, 0.00012630110722966492, 0.00012525123020168394, 0.00012363542919047177, 0.00012233770394232124 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.2648735139518976, "first_to_last_ratio": 2.3520205903150884, "imbalance_abs_log_ratio": 0.855274784383728, "last_quartile_mean": 0.1126153040677309, "mean": 0.1660086705815047, "normalized": [ 2.174595632268317, 1.943389981095013, 1.7490168198184282, 1.5798016086666014, 1.4557873374041121, 1.3727712901980273, 1.2902112306135503, 1.1987480127171795, 1.1282731510481394, 1.0677025559476454, 1.0238711946273023, 0.9743826246747078, 0.9332091864770874, 0.9012739856841682, 0.8680163310272434, 0.8423074257801569, 0.8168973360433722, 0.7977426245274992, 0.7748904637775177, 0.7599280557788093, 0.7477886868929706, 0.7367391104705062, 0.7235805912981054, 0.7121146568868953, 0.7033711391232357, 0.6946931471398464, 0.6884857463566807, 0.682653951912613, 0.6746941853139277, 0.6690857619715078, 0.6604542663637077, 0.6535219080951254 ], "peak_layer": 1, "peak_normalized": 2.174595632268317, "population_cv": 0.39609818748801673, "rest_mean": 0.1744714594549603, "spike_contrast": 0.6895661390160055, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12030961066484451, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.36100172996520996, 0.32261958718299866, 0.2903519570827484, 0.262260764837265, 0.2416733205318451, 0.2278919368982315, 0.21418625116348267, 0.19900256395339966, 0.18730312585830688, 0.17724788188934326, 0.1699714958667755, 0.16175596415996552, 0.1549208164215088, 0.14961929619312286, 0.14409823715686798, 0.1398303359746933, 0.13561204075813293, 0.13243219256401062, 0.12863853573799133, 0.12615464627742767, 0.12413940578699112, 0.12230508029460907, 0.12012065201997757, 0.11821720749139786, 0.11676570773124695, 0.11532508581876755, 0.11429460346698761, 0.11332647502422333, 0.1120050847530365, 0.1110740378499031, 0.10964113473892212, 0.10849030315876007 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010120356273546349, "first_to_last_ratio": 2.7021644745482813, "imbalance_abs_log_ratio": 0.9940531090948486, "last_quartile_mean": 3.7452776723512216e-05, "mean": 6.280785908074904e-05, "normalized": [ 1.9944873163456722, 1.8472744197374709, 1.7373237819064158, 1.6248552595134786, 1.5471235814302835, 1.4758134067763151, 1.3807453375978929, 1.2829373095670376, 1.18406084586495, 1.104394561286083, 1.0901943179184503, 1.04561090576924, 0.9888677388041794, 0.9664019181449638, 0.9198712145053942, 0.9041143079769479, 0.8692998477179782, 0.8446721332610243, 0.7923141515826202, 0.7644051939108963, 0.7503405467650113, 0.7300062685920449, 0.7090277780404898, 0.6754004942706008, 0.6572075474610624, 0.6267776932652476, 0.618543683654507, 0.6069002486680706, 0.5945805123210514, 0.585842809953234, 0.5555319861870183, 0.5250728812043677 ], "peak_layer": 1, "peak_normalized": 1.9944873163456722, "population_cv": 0.404835319326031, "rest_mean": 6.624604820562699e-05, "spike_contrast": 0.6678381428743124, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.424163780640811e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001252694783033803, 0.00011602335143834352, 0.00010911758727161214, 0.00010205368016613647, 9.717151988297701e-05, 9.269268048228696e-05, 8.672165859024972e-05, 8.057854574872181e-05, 7.436832675011829e-05, 6.936465797480196e-05, 6.847277109045535e-05, 6.567258242284879e-05, 6.210866558831185e-05, 6.069763549021445e-05, 5.7775141613092273e-05, 5.678548404830508e-05, 5.459886233438738e-05, 5.305204831529409e-05, 4.9763555580284446e-05, 4.801065369974822e-05, 4.7127283323789015e-05, 4.585013084579259e-05, 4.4532516767503694e-05, 4.242045906721614e-05, 4.127779902773909e-05, 3.936656503356062e-05, 3.884940451825969e-05, 3.811810529441573e-05, 3.734432903002016e-05, 3.679553265101276e-05, 3.489177470328286e-05, 3.297870352980681e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0002129299391526729, "first_to_last_ratio": 1.6819035009332732, "imbalance_abs_log_ratio": 0.5199261882838202, "last_quartile_mean": 0.00012660056836466538, "mean": 0.0001612697526525153, "normalized": [ 1.5193801883639617, 1.4431946098051458, 1.385412670946279, 1.330312377973009, 1.2819779644638467, 1.2449644115522398, 1.2025034693572292, 1.15492638462487, 1.1161420015875916, 1.082061394531008, 1.0617850512168836, 1.0341029835254725, 1.0077924632434987, 0.9864749169321705, 0.9634524108641764, 0.9447429702319522, 0.9240217763435827, 0.9078672036949645, 0.8863448276060427, 0.8724873261680857, 0.860780266694715, 0.8496270597212868, 0.835986840267667, 0.8234693045553775, 0.8139348844586144, 0.8040606454444867, 0.7969452920911189, 0.7902469974302787, 0.780967396404848, 0.7743340697791917, 0.7639952190801171, 0.7557046210402876 ], "peak_layer": 1, "peak_normalized": 1.5193801883639617, "population_cv": 0.21463676643173402, "rest_mean": 0.00016614488736700473, "spike_contrast": 0.8122069076744362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001349440251942724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024503006716258824, 0.00023274363775271922, 0.000223425158765167, 0.0002145391481462866, 0.00020674426923505962, 0.000200775102712214, 0.00019392743706703186, 0.00018625469238031656, 0.00017999994452111423, 0.00017450377345085144, 0.00017123381257988513, 0.00016676953237038106, 0.00016252644127234817, 0.00015908856585156173, 0.0001553757319925353, 0.00015235846512950957, 0.0001490167633164674, 0.00014641151938121766, 0.00014294061111286283, 0.00014070581528358161, 0.00013881782069802284, 0.00013701914576813579, 0.00013481939095072448, 0.00013280069106258452, 0.00013126307749189436, 0.00012967066140845418, 0.0001285231701331213, 0.00012744293780997396, 0.0001259464188478887, 0.00012487666390370578, 0.00012320932000875473, 0.00012187229731353 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016320142822223715, "first_to_last_ratio": 1.2889451135618213, "imbalance_abs_log_ratio": 0.25382414241573026, "last_quartile_mean": 0.0001266162744286703, "mean": 0.00014373415069712792, "normalized": [ 1.1884743462891088, 1.1713432082551107, 1.1549462763142666, 1.141829986160868, 1.1273728485863819, 1.1144091299418613, 1.0997826147068885, 1.0853573683237072, 1.068690221812727, 1.0554384700276902, 1.0473609881004726, 1.0364070219594264, 1.025017513262844, 1.015761575885501, 1.006046000385858, 0.9971413723222552, 0.9860728003762654, 0.9774913363845525, 0.9641287247440273, 0.9550391277708558, 0.9473133601248869, 0.9403037775179377, 0.9293980025104189, 0.9176260020019316, 0.9094819029195437, 0.8991813518536786, 0.8931890476284186, 0.8860794380468366, 0.8769637206693581, 0.8703930215998912, 0.8602546600212342, 0.8517047834951949 ], "peak_layer": 1, "peak_normalized": 1.1884743462891088, "population_cv": 0.09781347741247146, "rest_mean": 0.00014562865706264146, "spike_contrast": 0.9167413819240867, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013350381632335485, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017082435078918934, 0.00016836202121339738, 0.00016600522212684155, 0.0001641199633013457, 0.00016204197891056538, 0.00016017864982131869, 0.0001580763200763613, 0.0001560029195388779, 0.00015360728139057755, 0.00015170255210250616, 0.00015054154209792614, 0.00014896708307787776, 0.00014733002171851695, 0.00014599962742067873, 0.00014460316742770374, 0.00014332326827570796, 0.00014173233648762107, 0.00014049888704903424, 0.00013857822341378778, 0.00013727173791266978, 0.0001361612812615931, 0.0001351537648588419, 0.00013358623255044222, 0.00013189419405534863, 0.00013072360889054835, 0.00012924306793138385, 0.00012838176917284727, 0.00012735987547785044, 0.00012604963558260351, 0.00012510520173236728, 0.00012364797294139862, 0.00012241906370036304 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00044128536683274433, "first_to_last_ratio": 3.0989416798064484, "imbalance_abs_log_ratio": 1.1310606595922121, "last_quartile_mean": 0.0001423987323505571, "mean": 0.0002548205006860371, "normalized": [ 2.1788974669107217, 1.9948922239742408, 1.9013861017158324, 1.7832746019711707, 1.6474456862757656, 1.56528080717771, 1.458232784418858, 1.3245892215128081, 1.2376166576486232, 1.160533346247024, 1.1299807802432356, 1.0675072615016112, 1.0048234765069712, 0.9484981041710848, 0.8855756138701967, 0.8439855543529297, 0.7905268728467926, 0.7470509271970999, 0.6989608308202429, 0.6713105901225432, 0.6509228727076454, 0.631461080612247, 0.611226351598172, 0.5954626643405772, 0.5833123929329473, 0.5733046444152264, 0.5652547933675338, 0.559916360487464, 0.5532517426009346, 0.5496220513891801, 0.5450027610740823, 0.5408933749885276 ], "peak_layer": 1, "peak_normalized": 2.1788974669107217, "population_cv": 0.4822174837001359, "rest_mean": 0.0002730129350244102, "spike_contrast": 0.5735309033794403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015658135525882245, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005552277434617281, 0.0005083394353277981, 0.0004845121584367007, 0.0004544149269349873, 0.00041980293462984264, 0.0003988656389992684, 0.00037158760824240744, 0.0003375324886292219, 0.00031537009635940194, 0.0002957276883535087, 0.00028794226818718016, 0.00027202273486182094, 0.00025604962138459086, 0.00024169676180463284, 0.0002256628213217482, 0.00021506482153199613, 0.0002014424535445869, 0.00019036389130633324, 0.00017810954886954278, 0.00017106370069086552, 0.0001658684923313558, 0.00016090922872535884, 0.00015575300494674593, 0.00015173609426710755, 0.00014863995602354407, 0.00014608977653551847, 0.00014403850946109742, 0.00014267816732171923, 0.00014097988605499268, 0.00014005496632307768, 0.0001388778764521703, 0.00013783072063233703 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0036361459642648697, 0.005152894649654627, 0.006386642809957266, 0.007284422870725393, 0.0037505170330405235, 0.0054016136564314365, 0.006585344672203064, 0.007922882214188576, 0.0044892593286931515, 0.006135277915745974, 0.007316684816032648, 0.0082859443500638, 0.0044554853811860085, 0.006464940495789051, 0.007893112488090992, 0.009196000173687935, 0.004961193539202213, 0.006907128728926182, 0.008217990398406982, 0.009676996618509293, 0.005381465889513493, 0.007502692751586437, 0.0091153085231781, 0.010543142445385456, 0.006136673036962748, 0.008469858206808567, 0.010564134456217289, 0.012301022186875343, 0.005773921962827444, 0.008156762458384037, 0.010457510128617287, 0.012218860909342766 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024784039669611957, "first_to_last_ratio": 4.996847129630424, "imbalance_abs_log_ratio": 1.6088071394647374, "last_quartile_mean": 4.959935540682636e-05, "mean": 0.0001217503483985638, "normalized": [ 2.9419062071052267, 2.5796112881191693, 2.2930860326581057, 2.0145554032379356, 1.8111535572920587, 1.6841756414503508, 1.5558994713918, 1.4047664015764878, 1.2749699043555138, 1.1699192416770665, 1.0919281315608147, 1.001760718810264, 0.9234449853785138, 0.8652188455092179, 0.7945394094365259, 0.7439156225148, 0.6942882953063699, 0.6562804150522287, 0.6082998698912725, 0.574440737760942, 0.551250788347098, 0.5265559301830911, 0.5020487822241618, 0.47689842352430634, 0.4572026867293029, 0.43893712719402944, 0.4296155316374904, 0.41603615348749273, 0.3999129464790325, 0.391194073390914, 0.3717285968379238, 0.3544587798804922 ], "peak_layer": 1, "peak_normalized": 2.9419062071052267, "population_cv": 0.6874423943830519, "rest_mean": 0.00013296059464833147, "spike_contrast": 0.4603997057303065, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.121501864981837e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00035817810567095876, 0.0003140685730613768, 0.00027918402338400483, 0.00024527282221242785, 0.00022050857660360634, 0.00020504897111095488, 0.0001894313027150929, 0.00017103079881053418, 0.00015522803005296737, 0.0001424380752723664, 0.000132942630443722, 0.00012196471652714536, 0.00011242974869674072, 0.00010534069588175043, 9.673544991528615e-05, 9.057198622031137e-05, 8.452984184259549e-05, 7.99023691797629e-05, 7.406072109006345e-05, 6.99383599567227e-05, 6.711497553624213e-05, 6.410836795112118e-05, 6.112461414886639e-05, 5.806254921481013e-05, 5.566458639805205e-05, 5.34407481609378e-05, 5.230584065429866e-05, 5.0652546633500606e-05, 4.868954056291841e-05, 4.762801472679712e-05, 4.5258086174726486e-05, 4.315547994337976e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00029608183103846386, "first_to_last_ratio": 2.343484699021421, "imbalance_abs_log_ratio": 0.8516390092453431, "last_quartile_mean": 0.0001263425492652459, "mean": 0.0001858347804954974, "normalized": [ 2.16920774571012, 1.9356424765570717, 1.751272112160408, 1.5771369240348272, 1.451727119925344, 1.369130935687747, 1.289826251583108, 1.2020808437231323, 1.127478492836466, 1.067970341065492, 1.0234512239081746, 0.9751296621832141, 0.9332398141881068, 0.9026389800429313, 0.8668613586828855, 0.8407822065969367, 0.8171794710239251, 0.7975274126338607, 0.777066300159633, 0.7612061150730458, 0.7492407752334779, 0.7365489154759011, 0.7249215428669998, 0.7138135717041164, 0.7048334785593375, 0.6962720074755282, 0.6911692191098191, 0.683337870004401, 0.6760758813563923, 0.6702804011180051, 0.662430102042749, 0.6545204472768429 ], "peak_layer": 1, "peak_normalized": 2.16920774571012, "population_cv": 0.39438035124793563, "rest_mean": 0.0001952685916977417, "spike_contrast": 0.6908033638721538, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001348922000033781, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00040311424527317286, 0.00035970969474874437, 0.00032544726855121553, 0.00029308689408935606, 0.00026978139067068696, 0.00025443214690312743, 0.00023969457834027708, 0.0002233884297311306, 0.0002095247182296589, 0.0001984660339076072, 0.0001901928335428238, 0.00018121300672646612, 0.0001734284160193056, 0.0001677417167229578, 0.00016109299031086266, 0.00015624657680746168, 0.0001518603676231578, 0.00014820833166595548, 0.00014440594532061368, 0.0001414585713064298, 0.00013923499500378966, 0.00013687640603166074, 0.00013471563579514623, 0.00013265138841234148, 0.00013098257477395236, 0.0001293915556743741, 0.00012844328011851758, 0.0001269879430765286, 0.0001256384130101651, 0.00012456141121219844, 0.0001231025526067242, 0.00012163266364950687 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.2625683266669512, "first_to_last_ratio": 2.343484691137257, "imbalance_abs_log_ratio": 0.8516390058810522, "last_quartile_mean": 0.1120418356731534, "mean": 0.16480014030821621, "normalized": [ 2.169207748902955, 1.9356423730461585, 1.7512721262865893, 1.5771370154883375, 1.4517271231757105, 1.3691309069442503, 1.2898263254837699, 1.202080803995401, 1.1274784933366124, 1.0679703770106845, 1.0234512066143264, 0.9751297116000902, 0.9332398524746174, 0.9026389696372329, 0.8668612939874407, 0.8407822044491968, 0.8171795266543764, 0.7975273711861923, 0.7770662316421936, 0.7612060929585753, 0.7492407781338722, 0.7365489418998509, 0.7249215250846839, 0.7138135688166896, 0.704833501980704, 0.6962720326387369, 0.6911692482611179, 0.6833378266924969, 0.6760758677045711, 0.6702804240542598, 0.6624300595814382, 0.6545204702768674 ], "peak_layer": 1, "peak_normalized": 2.169207748902955, "population_cv": 0.3943803506949354, "rest_mean": 0.17316613815448903, "spike_contrast": 0.6908033707584418, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11962375193834304, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.35748574137687683, 0.3189941346645355, 0.28860989212989807, 0.2599124014377594, 0.23924483358860016, 0.22563296556472778, 0.21256355941295624, 0.19810308516025543, 0.18580861389636993, 0.1760016679763794, 0.16866490244865417, 0.16070151329040527, 0.15379805862903595, 0.1487550288438797, 0.1428588628768921, 0.13856102526187897, 0.13467130064964294, 0.13143262267112732, 0.12806062400341034, 0.1254468709230423, 0.12347498536109924, 0.12138336896896362, 0.11946716904640198, 0.11763657629489899, 0.11615666002035141, 0.11474572867155075, 0.11390478909015656, 0.11261416971683502, 0.11141739785671234, 0.11046230792999268, 0.10916856676340103, 0.10786506533622742 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010104048214998329, "first_to_last_ratio": 2.7014816335786884, "imbalance_abs_log_ratio": 0.9938003756776099, "last_quartile_mean": 3.740187639778014e-05, "mean": 6.270627807225537e-05, "normalized": [ 1.9907250147127677, 1.8444603036108247, 1.7429056467158477, 1.6239117898045463, 1.5436487234069576, 1.4786997242989472, 1.3841625886805662, 1.282123032540977, 1.1824417181452251, 1.1042125830371328, 1.090862943651666, 1.0498276343781445, 0.9920003536086259, 0.9660066676365643, 0.9202805760437535, 0.8989382213875909, 0.8686500616317681, 0.8410510099026279, 0.7938571089189717, 0.7653969230084752, 0.751185801344563, 0.7265337395686793, 0.7106236720469997, 0.6758027080339929, 0.6556269427545545, 0.6278192648368973, 0.6205119527339269, 0.6073894466988616, 0.5911483388457922, 0.586775311152339, 0.5577767930941091, 0.5246434037673043 ], "peak_layer": 1, "peak_normalized": 1.9907250147127677, "population_cv": 0.4046861823965828, "rest_mean": 6.614403860617636e-05, "spike_contrast": 0.6673673413246971, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.414237118908204e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012483095633797348, 0.00011565924069145694, 0.00010929112613666803, 0.0001018294642562978, 9.67964660958387e-05, 9.272375609725714e-05, 8.679568418301642e-05, 8.039716340135783e-05, 7.41465191822499e-05, 6.924106128280982e-05, 6.84039550833404e-05, 6.583078356925398e-05, 6.220465002115816e-05, 6.0574682720471174e-05, 5.770736970589496e-05, 5.636907008010894e-05, 5.446981231216341e-05, 5.273917849990539e-05, 4.977982462150976e-05, 4.799519228981808e-05, 4.7104065743042156e-05, 4.555822670226917e-05, 4.456056558410637e-05, 4.237707253196277e-05, 4.1111925384029746e-05, 3.9368209399981424e-05, 3.89099950552918e-05, 3.808713154285215e-05, 3.706871211761609e-05, 3.679449582705274e-05, 3.4976106690010056e-05, 3.2898435165407136e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021103927974763792, "first_to_last_ratio": 1.675431813331652, "imbalance_abs_log_ratio": 0.5160709310565479, "last_quartile_mean": 0.00012596112719620578, "mean": 0.00016011051866371417, "normalized": [ 1.5150472374771589, 1.439502583502474, 1.3841101446264577, 1.327205260326266, 1.2794452130965426, 1.2428385988073838, 1.2018548814818522, 1.154676418187547, 1.1148050630726085, 1.0814152014364051, 1.0614838409967562, 1.034687172582471, 1.0074911480481314, 0.9875049829338368, 0.9623163727426111, 0.9431041078017948, 0.9243534563643369, 0.9073040220081667, 0.888778951337658, 0.8738270000353379, 0.8622815727966864, 0.8493922949455803, 0.8374788674952537, 0.8253865750371285, 0.8155793557629833, 0.8058361207192702, 0.7999903796437624, 0.7910448570865577, 0.7825332275919701, 0.7756749178925374, 0.766226246614208, 0.7568239275482656 ], "peak_layer": 1, "peak_normalized": 1.5150472374771589, "population_cv": 0.21304076077436968, "rest_mean": 0.0001649131305317429, "spike_contrast": 0.8136187466924137, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001341764145763591, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024257499899249524, 0.00023047950526233763, 0.00022161059314385056, 0.00021249952260404825, 0.00020485263667069376, 0.00019899153267033398, 0.00019242960843257606, 0.00018487584020476788, 0.00017849201685748994, 0.00017314594879280776, 0.00016995472833514214, 0.00016566429985687137, 0.00016130993026308715, 0.00015810993500053883, 0.00015407697355840355, 0.00015100088785402477, 0.00014799871132709086, 0.00014526891754940152, 0.00014230285887606442, 0.00013990889419801533, 0.00013806034985464066, 0.00013599664089269936, 0.00013408917584456503, 0.00013215307262726128, 0.00013058283366262913, 0.00012902283924631774, 0.00012808687461074442, 0.0001266546023543924, 0.00012529180094134063, 0.00012419371341820806, 0.00012268088175915182, 0.00012117547157686204 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016164389126060996, "first_to_last_ratio": 1.2829777917904304, "imbalance_abs_log_ratio": 0.24918377588932214, "last_quartile_mean": 0.00012599118417710997, "mean": 0.0001426773092134681, "normalized": [ 1.1847160153617624, 1.1696203114063919, 1.15376099486318, 1.1365907777467705, 1.126248503505431, 1.112765799827437, 1.097614510342432, 1.0821504120145113, 1.06828860482523, 1.0536786878013262, 1.046275103221197, 1.0365937358276223, 1.0246881313142864, 1.0168969779036778, 1.0047039605050023, 0.9958410772863212, 0.9859208451030141, 0.9762312163738422, 0.965237112554715, 0.9568250330809598, 0.9498426746244831, 0.9400597234006037, 0.9304691287067991, 0.9205814319068928, 0.9114724424808572, 0.9022957305012854, 0.8969570718532309, 0.8872712168205663, 0.8789155388108255, 0.8714969614364647, 0.8626996589435726, 0.8532906096493081 ], "peak_layer": 1, "peak_normalized": 1.1847160153617624, "population_cv": 0.0960837578275494, "rest_mean": 0.00014451401323269778, "spike_contrast": 0.9186590596986468, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013275910750962794, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000169032093253918, 0.00016687827883288264, 0.0001646155142225325, 0.00016216571384575218, 0.0001606901059858501, 0.00015876643010415137, 0.00015660468488931656, 0.00015439830895047635, 0.00015242054359987378, 0.00015033603995107114, 0.000149279716424644, 0.00014789840497542173, 0.00014619974535889924, 0.0001450881245546043, 0.00014334845764096826, 0.00014208392531145364, 0.00014066853327676654, 0.00013928604312241077, 0.0001377174339722842, 0.00013651722110807896, 0.00013552099699154496, 0.00013412519183475524, 0.00013275683159008622, 0.00013134608161635697, 0.0001300464355153963, 0.000128737126942724, 0.00012797542149201035, 0.00012659346975851804, 0.00012540130410343409, 0.00012434284144546837, 0.00012308766599744558, 0.00012174520816188306 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00043632716551655903, "first_to_last_ratio": 3.082278345441195, "imbalance_abs_log_ratio": 1.1256690460714511, "last_quartile_mean": 0.00014155994904285762, "mean": 0.00025262987765017897, "normalized": [ 2.1611427674158366, 1.9871567751170711, 1.8899978819847916, 1.7784604477508343, 1.6406551319992353, 1.5724690971097588, 1.4592296472822097, 1.3280082080145534, 1.2379389344592089, 1.1701308820499392, 1.131654892203279, 1.0685236385646504, 1.0072825277339317, 0.9454673581714634, 0.8865158004683366, 0.8408935109532261, 0.7895809204213243, 0.7481431077843942, 0.7047226439788088, 0.6727720648712864, 0.6548656488865329, 0.6315662143568483, 0.6127430670138975, 0.5973169234484098, 0.5839093169225268, 0.5733930286828336, 0.5691911560855093, 0.5599482690481828, 0.5552538441515218, 0.5513421690146183, 0.547054988271426, 0.542669135783553 ], "peak_layer": 1, "peak_normalized": 2.1611427674158366, "population_cv": 0.47902914554115894, "rest_mean": 0.0002705909153317205, "spike_contrast": 0.5751866206559582, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001556402741698548, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000545969232916832, 0.0005020151729695499, 0.0004774699336849153, 0.0004492922453209758, 0.00041447850526310503, 0.0003972526756115258, 0.00036864500725641847, 0.00033549455110915005, 0.00031274036155082285, 0.00029561002156697214, 0.00028588983695954084, 0.0002699409960769117, 0.00025446966174058616, 0.00023885330301709473, 0.00022396037820726633, 0.00021243482478894293, 0.00019947173132095486, 0.00018900330178439617, 0.0001780339953256771, 0.00016996232443489134, 0.00016543862875550985, 0.0001595524954609573, 0.00015479720605071634, 0.0001509001012891531, 0.00014751293929293752, 0.0001448562106816098, 0.00014379469212144613, 0.00014145966270007193, 0.0001402737107127905, 0.00013928550470154732, 0.00013820243475493044, 0.00013709443737752736 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.027978340163826942, 0.0140878576785326, 0.014187770895659924, 0.014337530359625816, 0.014477464370429516, 0.009757626801729202, 0.009825476445257664, 0.00987118761986494, 0.009999874979257584, 0.007595349568873644, 0.007658981718122959, 0.007689184043556452, 0.007755559403449297, 0.00625597033649683, 0.006315375678241253, 0.00635211868211627, 0.006408242508769035, 0.005408597644418478, 0.005475745070725679, 0.005517482291907072, 0.005585856735706329, 0.004830927588045597, 0.004899685736745596, 0.004948681220412254, 0.004989070352166891, 0.004419817589223385, 0.004463871940970421, 0.004542885348200798, 0.004592607729136944, 0.004137600306421518, 0.004193917382508516, 0.004230021964758635 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024312340610777028, "first_to_last_ratio": 2.286272500967691, "imbalance_abs_log_ratio": 0.8269227625411849, "last_quartile_mean": 0.00010634051977831405, "mean": 0.00017363850702167838, "normalized": [ 1.061084599304074, 1.6344608012282997, 1.6065574901355, 1.6039206242202213, 1.1359733847422597, 1.548756457352017, 1.2967221414405534, 1.313883475593909, 1.1957449434685963, 1.344067980756563, 1.165161355052429, 1.1171979507437393, 1.0391889074947966, 1.1329181602378293, 1.0774500297071945, 1.0521010358096583, 0.862761425176938, 0.9019434894037968, 0.9280163978632505, 0.8870730707939211, 0.8354471012965897, 0.8049103603266032, 0.8255366455754076, 0.7297241480789424, 0.6373588767497196, 0.6752584188072907, 0.6298229328646214, 0.6250051046248064, 0.6032481952321226, 0.5674884668477911, 0.6733729976615692, 0.4878430314089892 ], "peak_layer": 2, "peak_normalized": 1.6344608012282997, "population_cv": 0.3218023265359089, "rest_mean": 0.00018114369622371332, "spike_contrast": 0.7348336602687936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001331104853306897, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.00018424514564685524, 0.0002838053333107382, 0.000278960244031623, 0.00027850238257087767, 0.00019724872254300863, 0.00026892375899478793, 0.00022516089666169137, 0.0002281407651025802, 0.0002076273667626083, 0.00023338195751421154, 0.00020231687813065946, 0.00019398858421482146, 0.0001804432104108855, 0.00019671821792144328, 0.00018708681454882026, 0.00018268525309395045, 0.000149808605783619, 0.00015661212091799825, 0.0001611393818166107, 0.00015403004363179207, 0.00014506578736472875, 0.00013976343325339258, 0.00014334495062939823, 0.0001267082116100937, 0.00011067004379583523, 0.00011725086369551718, 0.00010936151375062764, 0.00010852495324797928, 0.00010474711598362774, 9.853785013547167e-05, 0.000116923481982667, 8.470833563478664e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00025287891548941843, "first_to_last_ratio": 2.2739168136310908, "imbalance_abs_log_ratio": 0.8215038131425536, "last_quartile_mean": 0.0001112085165004828, "mean": 0.0001807239029858465, "normalized": [ 1.0592123382591336, 1.645736468094544, 1.60181949161791, 1.5891655894821897, 1.1402836456293184, 1.5483130487587196, 1.300612463539172, 1.3089009659072275, 1.1885078058073624, 1.3408206015264876, 1.166937022076678, 1.1148751106540256, 1.0356001272424655, 1.1351884917697168, 1.0748875173492451, 1.0458945475116934, 0.8650051516349683, 0.9033374117584585, 0.9274894298403746, 0.8851829368231718, 0.8352656795149113, 0.8098299287299446, 0.8239713600606818, 0.7303600865627311, 0.6419462275110265, 0.6820467100618508, 0.635541252420758, 0.625618271589476, 0.6029839777576883, 0.5727133544881818, 0.6681570652306708, 0.4937959207892163 ], "peak_layer": 2, "peak_normalized": 1.645736468094544, "population_cv": 0.3200470231318611, "rest_mean": 0.00018847914530110925, "spike_contrast": 0.7366629038009043, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001388455944834277, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001914249878609553, 0.00029742391780018806, 0.00028948707040399313, 0.0002872002078220248, 0.00020607651094906032, 0.0002798171772155911, 0.00023505176068283617, 0.00023654969118069857, 0.00021479176939465106, 0.0002423183323116973, 0.00021089341316837817, 0.00020148458133917302, 0.00018715769692789763, 0.00020515569485723972, 0.0001942578674061224, 0.0001890181447379291, 0.00015632710710633546, 0.00016325466276612133, 0.00016761950973886997, 0.0001599737151991576, 0.00015095247363206, 0.0001463556254748255, 0.00014891132013872266, 0.00013199352542869747, 0.0001160150277428329, 0.0001232621434610337, 0.00011485749564599246, 0.00011306417582090944, 0.00010897361789830029, 0.0001035029927152209, 0.00012075195263605565, 8.924072608351707e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.22425555251538754, "first_to_last_ratio": 2.2739168285843645, "imbalance_abs_log_ratio": 0.8215038197185521, "last_quartile_mean": 0.09862082451581955, "mean": 0.1602677651681006, "normalized": [ 1.0592124025748084, 1.645736496509649, 1.6018195299533093, 1.5891655927499475, 1.1402836784663197, 1.5483129508886688, 1.300612441604666, 1.3089009387606192, 1.1885077887054891, 1.3408205692725423, 1.1669370180912673, 1.1148751103389594, 1.0356000505037584, 1.1351884646301031, 1.0748875233039092, 1.0458946120331756, 0.8650051766933494, 0.9033374073089909, 0.9274894888798537, 0.8851828780404403, 0.835265755835364, 0.8098299455472033, 0.8239713233255873, 0.730360099613448, 0.6419462245963619, 0.6820467317825786, 0.6355412485786012, 0.6256182682629655, 0.602983984145969, 0.5727133411523797, 0.6681570418174577, 0.49379591603225714 ], "peak_layer": 2, "peak_normalized": 1.645736496509649, "population_cv": 0.3200470194234096, "rest_mean": 0.16714519124340127, "spike_contrast": 0.7366629183018027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1231296643614769, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.16975760459899902, 0.26375851035118103, 0.25672003626823425, 0.254692018032074, 0.18275071680545807, 0.24814465641975403, 0.20844624936580658, 0.20977462828159332, 0.19047948718070984, 0.21489031612873077, 0.1870223879814148, 0.17867854237556458, 0.16597330570220947, 0.18193411827087402, 0.1722698211669922, 0.16762319207191467, 0.13863244652748108, 0.1447758674621582, 0.14864666759967804, 0.1418662816286087, 0.13386617600917816, 0.12978963553905487, 0.13205604255199432, 0.11705318093299866, 0.10288328677415848, 0.1093101054430008, 0.1018567755818367, 0.10026644170284271, 0.09663889557123184, 0.09178748726844788, 0.10708403587341309, 0.07913956791162491 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.143860511860112e-05, "first_to_last_ratio": 0.9636580872932836, "imbalance_abs_log_ratio": 0.037018728516889636, "last_quartile_mean": 7.413273033307632e-05, "mean": 7.686168260079285e-05, "normalized": [ 0.586022200160155, 0.9787081875553917, 0.912570839911241, 1.015732496910769, 0.8201270226617754, 1.1119332047208261, 1.0008210861718638, 1.0096344009845781, 0.9335569648148418, 1.090167719321714, 1.035494250546431, 0.9697380161483544, 1.0399314841024359, 1.0649519568255064, 1.06563466639597, 1.1389203959735625, 0.9952637999761301, 1.0027093290728673, 1.1427249020241628, 1.1090175842590573, 1.0432879501645442, 1.0450477353718262, 1.1125450117027875, 1.0594965280199307, 0.9180884623091344, 0.995296932026832, 0.9631737996002885, 0.9446177686211702, 0.9674620335911367, 0.9355074959711648, 1.1351899163905306, 0.8566258576930204 ], "peak_layer": 19, "peak_normalized": 1.1427249020241628, "population_cv": 0.10846075768175395, "rest_mean": 7.635363915512614e-05, "spike_contrast": 1.0425844542348655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.960511720739305e-05, "top_five_layers": [ 19, 16, 31, 23, 6 ], "top_five_spike_overlap": 1 }, "values": [ 4.504265234572813e-05, 7.522515807067975e-05, 7.014173024799675e-05, 7.807090878486633e-05, 6.303634290816262e-05, 8.546505705453455e-05, 7.692479266552255e-05, 7.760219887131825e-05, 7.175475911935791e-05, 8.379212522413582e-05, 7.958983042044565e-05, 7.453569560311735e-05, 7.993088365765288e-05, 8.185399929061532e-05, 8.190647349692881e-05, 8.753933798288926e-05, 7.64976502978243e-05, 7.706992619205266e-05, 8.78317587194033e-05, 8.52409575600177e-05, 8.018886728677899e-05, 8.032412733882666e-05, 8.551208156859502e-05, 8.143468585330993e-05, 7.056582398945466e-05, 7.650019688298926e-05, 7.403115887427703e-05, 7.260491111082956e-05, 7.436075975419953e-05, 7.190468022599816e-05, 8.725260704522952e-05, 6.584170478163287e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 8.107683152047684e-05, "first_to_last_ratio": 1.03534602173519, "imbalance_abs_log_ratio": 0.034735691361772646, "last_quartile_mean": 7.83089226388256e-05, "mean": 8.487996228723205e-05, "normalized": [ 0.5954853629443996, 1.025163681929567, 0.9520754208460104, 1.0126519094835282, 0.8443534832663374, 1.1548844288399933, 1.0326421147312754, 1.0242956755731092, 0.9710206925084107, 1.15116021353655, 1.072832454869962, 1.0193061391332177, 1.0419196505755806, 1.0947619162703996, 1.1089694127677372, 1.1239932261258834, 0.9835832115239759, 1.0364664516448652, 1.11612142168979, 1.093437276508762, 1.0179007506072324, 1.033774483284387, 1.0848380474290875, 1.0276879808812729, 0.8974590825111308, 0.9692041024722606, 0.9378977551023808, 0.9155031739616566, 0.918412101050971, 0.8874638084461439, 1.0536745962271137, 0.8010599732570078 ], "peak_layer": 6, "peak_normalized": 1.1548844288399933, "population_cv": 0.11200209045145568, "rest_mean": 8.468612051931107e-05, "spike_contrast": 1.0146492401244358, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.592670783400536e-05, "top_five_layers": [ 6, 10, 16, 19, 15 ], "top_five_spike_overlap": 0 }, "values": [ 5.054477514931932e-05, 8.701585466042161e-05, 8.081212581600994e-05, 8.59538558870554e-05, 7.166869181673974e-05, 9.802654676605016e-05, 8.76506237545982e-05, 8.694217831362039e-05, 8.242019976023585e-05, 9.771043551154435e-05, 9.106197830988094e-05, 8.651866664877161e-05, 8.843810064718127e-05, 9.292335016652942e-05, 9.41292819334194e-05, 9.540450264466926e-05, 8.348650590050966e-05, 8.797523332759738e-05, 9.473634418100119e-05, 9.281091479351744e-05, 8.639937732368708e-05, 8.774673915468156e-05, 9.20810125535354e-05, 8.723011706024408e-05, 7.617629307787865e-05, 8.226600766647607e-05, 7.960872608236969e-05, 7.770787487970665e-05, 7.795478450134397e-05, 7.532789459219202e-05, 8.943585999077186e-05, 6.799394031986594e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.1427921979011444e-05, "first_to_last_ratio": 0.40836765899304117, "imbalance_abs_log_ratio": 0.8955873853644011, "last_quartile_mean": 5.2472132665570825e-05, "mean": 3.864622581772892e-05, "normalized": [ 0.2626641640048693, 0.5716512870137274, 0.5157707114648397, 0.4843895386483896, 0.5109475889421058, 0.7512596968779521, 0.675728069412672, 0.6632970161544233, 0.7004288748821966, 0.9130902556041411, 0.8717836320781894, 0.8044820763577291, 0.9241814793896349, 0.9531622930600316, 1.0319070449118461, 1.0273708471578158, 0.9893551980130566, 1.0844069350216101, 1.1887300698220302, 1.2100415764612447, 1.1128447755732793, 1.2346303137099222, 1.3033621630444512, 1.3524689418392815, 1.2055635543985064, 1.318388624044801, 1.3615446320871405, 1.2933406942036905, 1.3753836688642536, 1.3573239762703455, 1.6442457326403273, 1.3062545680454958 ], "peak_layer": 31, "peak_normalized": 1.6442457326403273, "population_cv": 0.32680115045148705, "rest_mean": 3.691592014042221e-05, "spike_contrast": 1.2999777952883038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.798987647518516e-05, "top_five_layers": [ 31, 29, 27, 30, 24 ], "top_five_spike_overlap": 1 }, "values": [ 1.0150978596357163e-05, 2.2092164726927876e-05, 1.99325913854409e-05, 1.8719827494351193e-05, 1.974619590328075e-05, 2.9033351893303916e-05, 2.6114339561900124e-05, 2.5633926270529628e-05, 2.7068932467955165e-05, 3.528749221004546e-05, 3.369114710949361e-05, 3.1090195989236236e-05, 3.571612614905462e-05, 3.683612521854229e-05, 3.9879312680568546e-05, 3.970400575781241e-05, 3.823484439635649e-05, 4.190823528915644e-05, 4.5939930714666843e-05, 4.676354001276195e-05, 4.300725049688481e-05, 4.771380190504715e-05, 5.037002847529948e-05, 5.226782013778575e-05, 4.65904813609086e-05, 5.09507444803603e-05, 5.261856131255627e-05, 4.998273652745411e-05, 5.3153387852944434e-05, 5.24554488947615e-05, 6.354389188345522e-05, 5.048180901212618e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031520502852799837, "first_to_last_ratio": 1.31300997816315, "imbalance_abs_log_ratio": 0.27232219480674863, "last_quartile_mean": 0.00024006293460843153, "mean": 0.00030979480334281106, "normalized": [ 0.607123837738274, 1.0354820546705408, 0.9347765087792952, 1.1085716606000595, 0.9306423427143409, 1.2565833735558338, 1.1285140374487672, 1.138017377238765, 1.0370737734279494, 1.2142990636699202, 1.1645184658462944, 1.1206382096550715, 1.1232577862254296, 1.1815362250572234, 1.1924463988431009, 1.2309950595128432, 1.0374639291264394, 1.065463636492051, 1.1630951913863559, 1.1082935818936954, 1.0270641612904527, 0.9934767629460804, 1.0585185268550952, 0.9428720751183239, 0.8088208526771234, 0.8683362720246933, 0.8018903045940483, 0.8027332776454693, 0.7640102071384416, 0.7171383971343437, 0.8334550561316563, 0.6028915925620211 ], "peak_layer": 6, "peak_normalized": 1.2565833735558338, "population_cv": 0.17765693833110424, "rest_mean": 0.0003117367305094376, "spike_contrast": 0.9601319554288655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00029930839664302764, "top_five_layers": [ 6, 16, 10, 15, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00018808380991686136, 0.00032078695949167013, 0.00028958890470676124, 0.00034342973958700895, 0.0002883081615436822, 0.00038928299909457564, 0.00034960778430104256, 0.0003525518695823848, 0.00032128006569109857, 0.00037618353962898254, 0.00036076176911592484, 0.00034716789377853274, 0.0003479794249869883, 0.00036603378248400986, 0.0003694136976264417, 0.00038135587237775326, 0.0003214009338989854, 0.0003300750977359712, 0.0003603208460845053, 0.000343343592248857, 0.00031817913986742496, 0.00030777393840253353, 0.00032792353886179626, 0.00029209686908870935, 0.0002505684969946742, 0.0002690060646273196, 0.00024842144921422005, 0.0002486825978849083, 0.00023668639187235385, 0.00022216574870981276, 0.000258200045209378, 0.0001867726823547855 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014031016267836094, 0.014151837676763535, 0.014265215955674648, 0.014401593245565891, 0.009722241200506687, 0.009786579757928848, 0.009839320555329323, 0.00996283907443285, 0.007562503684312105, 0.007635751739144325, 0.007660485804080963, 0.007726665120571852, 0.006238248664885759, 0.006290202960371971, 0.006327399518340826, 0.006382675375789404, 0.005390914157032967, 0.0054517327807843685, 0.005509276874363422, 0.0055727302096784115, 0.0048178439028561115, 0.004884926602244377, 0.004938251338899136, 0.004975184798240662, 0.004412420094013214, 0.004455919377505779, 0.004530393984168768, 0.0045819394290447235, 0.004127359017729759, 0.004185030702501535, 0.004221425391733646, 0.004281661007553339 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.6193598993559135e-05, "first_to_last_ratio": 1.5723739147519853, "imbalance_abs_log_ratio": 0.4525865249799748, "last_quartile_mean": 3.573806361600873e-05, "mean": 4.4294749727669114e-05, "normalized": [ 1.5642840408451666, 1.493254721227763, 1.1816235069560812, 1.0717429494942774, 1.3553344243310417, 1.3257825682631335, 1.1255110934944355, 1.0314977817441078, 1.194075821513187, 1.0893775754580661, 1.0407197138445958, 0.9875719064328349, 1.135803276632048, 1.0324555952346264, 0.9635988909717662, 0.90208003019059, 0.9904506854292476, 0.9196323607404276, 0.8708711781381843, 0.8075366439875266, 0.9218591005239072, 0.8776357461813653, 0.8517186042296377, 0.8109904231766266, 0.8722342267309339, 0.8500552841464633, 0.7844009556175089, 0.7665260139039981, 0.8384393111065888, 0.8154485842192296, 0.7754690286134897, 0.7520179566211438 ], "peak_layer": 1, "peak_normalized": 1.5642840408451666, "population_cv": 0.20749976292329902, "rest_mean": 4.538663482957485e-05, "spike_contrast": 0.8460325450777163, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.839857017737813e-05, "top_five_layers": [ 1, 2, 5, 6, 9 ], "top_five_spike_overlap": 0 }, "values": [ 6.928957009222358e-05, 6.614334415644407e-05, 5.23397175129503e-05, 4.747258572024293e-05, 6.003419912303798e-05, 5.872520705452189e-05, 4.985423220205121e-05, 4.5689936087001115e-05, 5.289128966978751e-05, 4.8253707063850015e-05, 4.6098419261397794e-05, 4.374425043351948e-05, 5.03101218782831e-05, 4.573236219584942e-05, 4.2682371713453904e-05, 3.9957409171620384e-05, 4.3871765228686854e-05, 4.073488526046276e-05, 3.857502088067122e-05, 3.576963354134932e-05, 4.083351814188063e-05, 3.887465572915971e-05, 3.7726662412751466e-05, 3.592261782614514e-05, 3.863539677695371e-05, 3.7652986065950245e-05, 3.474484401522204e-05, 3.395307794562541e-05, 3.713845944730565e-05, 3.612009095377289e-05, 3.4349206543993205e-05, 3.3310447179246694e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 9.100913848669734e-05, "first_to_last_ratio": 0.8419197805282016, "imbalance_abs_log_ratio": 0.17207054181035333, "last_quartile_mean": 0.00010809716150106397, "mean": 9.889523016681778e-05, "normalized": [ 1.0744641570554045, 0.9868986027413778, 0.8415148680287817, 0.7657672453742986, 1.0045876840265406, 0.9858056114400382, 0.8710937611903306, 0.8319330217638002, 1.0050985706445066, 0.9363425314153265, 0.9111662095698898, 0.8822483642571924, 1.0547921529408322, 0.9937743096656301, 0.9653849347701057, 0.9236053864198286, 1.0608213360421703, 1.0216834776263237, 0.9948854733452297, 0.9444045194208119, 1.0902032759381237, 1.070297753084291, 1.0352646449837608, 1.0035839360241054, 1.109605122267389, 1.0988987221493531, 1.0612028824132491, 1.0392747076439328, 1.1393068813062424, 1.1142247321092786, 1.0935193303224786, 1.0883457940193757 ], "peak_layer": 29, "peak_normalized": 1.1393068813062424, "population_cv": 0.09089893356811851, "rest_mean": 9.776359500527313e-05, "spike_contrast": 1.0740814107080978, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010500606003915891, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.00010625938011799008, 9.759956446941942e-05, 8.322180656250566e-05, 7.573072798550129e-05, 9.934893023455516e-05, 9.749147284310311e-05, 8.614701800979674e-05, 8.227420767070726e-05, 9.939945448422804e-05, 9.259981015929952e-05, 9.010999201564118e-05, 8.725015504751354e-05, 0.00010431391274323687, 9.827953908825293e-05, 9.547196532366797e-05, 9.134016727330163e-05, 0.00010491017019376159, 0.00010103962267749012, 9.838942787609994e-05, 9.339710231870413e-05, 0.0001078159039025195, 0.00010584734263829887, 0.00010238273534923792, 9.924966434482485e-05, 0.00010973465396091342, 0.00010867584205698222, 0.00010494790330994874, 0.00010277931141899899, 0.00011267201625742018, 0.000110191511339508, 0.00010814384586410597, 0.0001076322078006342 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.08070781361311674, "first_to_last_ratio": 0.8419197775247833, "imbalance_abs_log_ratio": 0.1720705453776983, "last_quartile_mean": 0.09586164355278015, "mean": 0.0877012787386775, "normalized": [ 1.074464079725317, 0.986898606287062, 0.8415149217948403, 0.7657672391640048, 1.0045876578450608, 0.9858055872189405, 0.8710938052118494, 0.8319330371629138, 1.0050985716236776, 0.9363425240221704, 0.9111662153888755, 0.8822483595926541, 1.0547922001671663, 0.993774279010469, 0.9653849224854107, 0.9236053602662656, 1.060821305580311, 1.021683475130349, 0.9948854782496288, 0.9444045803369281, 1.0902032654624292, 1.07029776221103, 1.0352646575744078, 1.0035839255047951, 1.109605076031185, 1.0988987393251362, 1.0612029192639736, 1.0392747445534645, 1.1393068879878778, 1.1142247084644445, 1.0935193626414328, 1.0883457447159286 ], "peak_layer": 29, "peak_normalized": 1.1393068879878778, "population_cv": 0.09089892559784572, "rest_mean": 0.08669773351263117, "spike_contrast": 1.0740813996684329, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0931204229593277, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.09423187375068665, 0.08655226975679398, 0.0738019347190857, 0.0671587660908699, 0.08810362219810486, 0.08645641058683395, 0.07639604061841965, 0.07296159118413925, 0.08814842998981476, 0.0821184366941452, 0.07991044223308563, 0.07737430930137634, 0.09250662475824356, 0.08715527504682541, 0.08466549217700958, 0.08100137114524841, 0.09303538501262665, 0.08960294723510742, 0.08725272864103317, 0.08282548934221268, 0.09561222046613693, 0.09386648237705231, 0.09079403430223465, 0.0880155935883522, 0.0973137840628624, 0.09637482464313507, 0.09306885302066803, 0.09114572405815125, 0.0999186709523201, 0.09771893173456192, 0.09590304642915726, 0.09544931352138519 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0474835562490625e-05, "first_to_last_ratio": 0.595870196340598, "imbalance_abs_log_ratio": 0.5177324270116084, "last_quartile_mean": 1.7579056020622374e-05, "mean": 1.45315484587627e-05, "normalized": [ 0.6891108502390517, 0.67218760206668, 0.6248497232673742, 0.6113105238885437, 0.8536003812586072, 0.8434482348779837, 0.7260190066641083, 0.7461464268756399, 0.9515026546685825, 0.8968707547028398, 0.8666952712848448, 0.9173542983424958, 1.0474470628626384, 1.0570493146141757, 0.9856965747654381, 1.0403130781822671, 1.05710126231986, 1.0899464822844398, 1.0639055355381295, 0.9944043245776518, 1.1636416255463544, 1.1162669452398895, 1.1491653646620976, 1.158233556446643, 1.241320970071456, 1.2666472916328237, 1.1327368714456698, 1.1012802199127885, 1.2948691657915095, 1.2555094531837419, 1.2151558495322814, 1.1702133232533922 ], "peak_layer": 29, "peak_normalized": 1.2948691657915095, "population_cv": 0.19478066534993646, "rest_mean": 1.408557606434139e-05, "spike_contrast": 1.202634475953173, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6939799388637765e-05, "top_five_layers": [ 29, 26, 30, 25, 31 ], "top_five_spike_overlap": 1 }, "values": [ 1.0013847713707946e-05, 9.767926712811459e-06, 9.080034033104312e-06, 8.883288501237985e-06, 1.2404135304677766e-05, 1.2256608897587284e-05, 1.0550180377322249e-05, 1.0842662959476002e-05, 1.3826806934957858e-05, 1.3032920833211392e-05, 1.2594424333656207e-05, 1.3330578440218233e-05, 1.522102775197709e-05, 1.5360563338617794e-05, 1.4323697541840374e-05, 1.5117359907890204e-05, 1.5361318219220266e-05, 1.5838610124774277e-05, 1.546019484521821e-05, 1.445023463020334e-05, 1.690951467026025e-05, 1.622108720766846e-05, 1.669915218371898e-05, 1.6830927052069455e-05, 1.8038315829471685e-05, 1.8406346498522907e-05, 1.6460420738440007e-05, 1.600330688233953e-05, 1.8816454030456953e-05, 1.8244496459374204e-05, 1.7658096112427302e-05, 1.7005011613946408e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.836043712610262e-05, "first_to_last_ratio": 0.6391636816233681, "imbalance_abs_log_ratio": 0.44759470463089907, "last_quartile_mean": 0.00010695294349716278, "mean": 9.121693358338234e-05, "normalized": [ 0.7254188545798665, 0.6902023390007371, 0.6496225556724757, 0.6277900804246503, 0.8680656002782756, 0.8588952084862168, 0.7971788633739051, 0.7782428701770085, 0.9734232055545582, 0.9237343700472408, 0.9096320021673064, 0.8918620199754489, 1.0844263684244682, 1.0342037426828012, 1.0147200800673386, 0.9791745321037664, 1.1290928536156495, 1.0928814238713915, 1.0660995437899934, 1.013549283136708, 1.170635644938427, 1.1503062295054498, 1.1129730585678779, 1.0777737723208793, 1.1919439416858, 1.1807148042349604, 1.1393605783805036, 1.1153320412921235, 1.222157091120308, 1.1946826876057626, 1.1714041050576325, 1.1645002478604691 ], "peak_layer": 29, "peak_normalized": 1.222157091120308, "population_cv": 0.17156610618115575, "rest_mean": 8.883977756744975e-05, "spike_contrast": 1.1712498490939807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010405357606941834, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 6.617048347834498e-05, 6.295814091572538e-05, 5.925657751504332e-05, 5.7265086070401594e-05, 7.91822822066024e-05, 7.834578718757257e-05, 7.271621143445373e-05, 7.098892820067704e-05, 8.879267988959327e-05, 8.426021668128669e-05, 8.297384192701429e-05, 8.135291864164174e-05, 9.891804802464321e-05, 9.433689410798252e-05, 9.255965414922684e-05, 8.931729826144874e-05, 0.00010299238783773035, 9.968929225578904e-05, 9.724633127916604e-05, 9.245285764336586e-05, 0.00010678179387468845, 0.00010492740693734959, 0.00010152198956348002, 9.831121860770509e-05, 0.00010872547136386856, 0.00010770118387881666, 0.0001039289782056585, 0.00010173716873396188, 0.00011148142220918089, 0.00010897529136855155, 0.00010685189045034349, 0.00010622214176692069 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.421776268121903e-05, "first_to_last_ratio": 0.5089233562442377, "imbalance_abs_log_ratio": 0.6754578508927926, "last_quartile_mean": 0.00010653423942130757, "mean": 8.540443025140121e-05, "normalized": [ 0.5542697148172375, 0.5395320644633284, 0.526750599523528, 0.5251350203104778, 0.7537064875235374, 0.7453106043033457, 0.7183783488935762, 0.7156010196903468, 0.915261682615852, 0.8828592813937833, 0.8733821992497209, 0.8643716402564832, 1.0724322757212523, 1.0352702478756863, 1.026729960575756, 1.0038080492636434, 1.1695135624376918, 1.1411903421843295, 1.1203495535033974, 1.0697851231751558, 1.23599072599408, 1.2140634563521424, 1.1785251437606976, 1.1385123493568967, 1.2633504856351931, 1.2489710766708417, 1.2127527443953423, 1.1862530196716095, 1.298334269521613, 1.274402551981099, 1.2503128844914202, 1.2448935143909348 ], "peak_layer": 29, "peak_normalized": 1.298334269521613, "population_cv": 0.24541141609846925, "rest_mean": 8.214501078193145e-05, "spike_contrast": 1.2539446328637511, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010300529538653791, "top_five_layers": [ 29, 30, 25, 31, 26 ], "top_five_spike_overlap": 1 }, "values": [ 4.73370891995728e-05, 4.6078428567852825e-05, 4.498683483689092e-05, 4.4848857214674354e-05, 6.436987314373255e-05, 6.365282752085477e-05, 6.135269359219819e-05, 6.111549737397581e-05, 7.816740253474563e-05, 7.540009391959757e-05, 7.459070911863819e-05, 7.382116746157408e-05, 9.159046749118716e-05, 8.841666567604989e-05, 8.768728730501607e-05, 8.572965452913195e-05, 9.98816394712776e-05, 9.746271098265424e-05, 9.56828152993694e-05, 9.136438893619925e-05, 0.00010555908374954015, 0.00010368639777880162, 0.00010065126843983307, 9.723399853101e-05, 0.0001078957284335047, 0.00010666766320355237, 0.00010357445717090741, 0.00010131126327905804, 0.00011088349856436253, 0.00010883962386287749, 0.00010678225953597575, 0.00010631942132022232 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014064586594031425, "first_to_last_ratio": 1.0563608173139223, "imbalance_abs_log_ratio": 0.054829809986132017, "last_quartile_mean": 0.00013314188072399702, "mean": 0.0001347423383322166, "normalized": [ 1.1217990599984589, 1.0213028410040081, 0.9566177087867302, 0.90173442925003, 1.162761655685041, 1.2034553361266669, 1.0271109783027095, 0.9557256445604886, 1.16094674777681, 1.05596774404836, 1.0011734009390578, 0.9818696062747876, 1.1123781705815696, 1.0319357929888284, 0.9826152250905227, 0.9109733958837348, 1.0154471845455557, 0.955267408663159, 0.9085227032176484, 0.853750823711138, 0.9825495622515935, 0.9600374683174735, 0.9243229318330685, 0.9067574744309754, 0.9933509912573504, 0.986398830187635, 0.9577583846169437, 0.9375602577665346, 1.0292265528944584, 1.007559112024919, 0.9944595916547317, 0.9986629853290112 ], "peak_layer": 6, "peak_normalized": 1.2034553361266669, "population_cv": 0.07980954026492859, "rest_mean": 0.0001359050228428613, "spike_contrast": 0.9452471975467027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012846384197473526, "top_five_layers": [ 6, 5, 9, 1, 13 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001511538284830749, 0.0001376127329422161, 0.00012889690697193146, 0.00012150180555181578, 0.00015667322441004217, 0.0001621563860680908, 0.00013839533494319767, 0.00012877670815214515, 0.00015642867947462946, 0.0001422835630364716, 0.00013490044511854649, 0.00013229940668679774, 0.000149884435813874, 0.00013904544175602496, 0.00013239987310953438, 0.0001227466855198145, 0.00013682372809853405, 0.0001287149643758312, 0.0001224164734594524, 0.00011503638233989477, 0.00013239102554507554, 0.0001293576933676377, 0.0001245454332092777, 0.00012217862240504473, 0.00013384643534664065, 0.000132909684907645, 0.00012905060430057347, 0.00012632906145881861, 0.00013868039241060615, 0.00013576087076216936, 0.00013399581075645983, 0.0001345621858490631 ] } } } } }, "uniform_value_backward": { "bits_per_byte": 8.038491115106986, "forward": { "activation_sha256": { "attention_branch_output": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", "mlp_branch_output": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", "post_attention_state": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", "post_mlp_state": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", "pre_attention_input": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", "pre_mlp_input": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd" }, "logits_sha256": "5e5a50bae6f74c41645e35a857f0cba662c8b3f5ba9a8a4635e860fdaa3b0c88", "loss_nats": 5.571857452392578, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.571857452392578, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_value_backward", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0023513585329055786, 0.0023208872880786657, 0.002778682392090559, 0.002497476991266012, 0.002628475893288851, 0.002920303260907531, 0.002932395786046982, 0.0032228652853518724, 0.003561305580660701, 0.003354643704369664, 0.0034162327647209167, 0.0036349669098854065, 0.0035210708156228065, 0.0038089773152023554, 0.003915763925760984, 0.0037244565319269896, 0.004162859171628952, 0.004167158622294664, 0.004440344404429197, 0.0041700867004692554, 0.004737259820103645, 0.004552462603896856, 0.004706811159849167, 0.005186605267226696, 0.005580419674515724, 0.004905648995190859, 0.005227713380008936, 0.005349088925868273, 0.005235649179667234, 0.004925078712403774, 0.005810525268316269, 0.005564783234149218 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024914435198297724, "first_to_last_ratio": 5.005386356076312, "imbalance_abs_log_ratio": 1.61051460380912, "last_quartile_mean": 4.977524895366514e-05, "mean": 0.00012233095264946314, "normalized": [ 2.946204230178919, 2.5883907308658176, 2.285503889899197, 2.0157125547168477, 1.8168856694000293, 1.687577213316311, 1.555459627812154, 1.3974018168018878, 1.2756749767931916, 1.1684180974434406, 1.0925035606597346, 1.0000212864934148, 0.922022236502042, 0.863306019148705, 0.7954975128821132, 0.747136294956448, 0.6955650087701328, 0.6567702950630189, 0.6067855157189815, 0.5734103460233209, 0.549704279492861, 0.527795032775898, 0.5009080420334019, 0.4762252632696638, 0.45754883501456406, 0.43797117166433, 0.4282553488048722, 0.41570642703193256, 0.40058854659018955, 0.39033614610916745, 0.3703926654711839, 0.3543213582962283 ], "peak_layer": 1, "peak_normalized": 2.946204230178919, "population_cv": 0.6883820030702144, "rest_mean": 0.00013360270127628205, "spike_contrast": 0.46004691130861425, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.146351006464101e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036041197017766535, 0.0003166403039358556, 0.00027958786813542247, 0.0002465840370859951, 0.00022226135479286313, 0.0002064429281745106, 0.00019028085807804018, 0.0001709454954834655, 0.00015605453518219292, 0.00014293369895312935, 0.00013364700134843588, 0.00012233355664648116, 0.0001127918585552834, 0.00010560904775047675, 9.731396858114749e-05, 9.139789472101256e-05, 8.508913015248254e-05, 8.03433358669281e-05, 7.42286501917988e-05, 7.014583388809115e-05, 6.724584818584844e-05, 6.456566916313022e-05, 6.127655797172338e-05, 5.825709013151936e-05, 5.597238487098366e-05, 5.3577430662699044e-05, 5.2388884796528146e-05, 5.085376324132085e-05, 4.900437852484174e-05, 4.775019260705449e-05, 4.5310487621463835e-05, 4.334446930442937e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002986813742609229, "first_to_last_ratio": 2.3520308544062645, "imbalance_abs_log_ratio": 0.8552791483203458, "last_quartile_mean": 0.0001269887143280357, "mean": 0.00018719729814620223, "normalized": [ 2.1746021240914017, 1.94336263487061, 1.7490368812517914, 1.5797937988337565, 1.4558020735282082, 1.372777068369229, 1.290232936323773, 1.1987380034727069, 1.1282687150208124, 1.0677045109732362, 1.0238712087163733, 0.9743741602296072, 0.9332108464677616, 0.9012687760530631, 0.8680188022864366, 0.8423083374767277, 0.8168968442148058, 0.7977443195284458, 0.774889242886524, 0.759928616461841, 0.7477907292059667, 0.7367409087088999, 0.7235787757136496, 0.7121132246295925, 0.7033660136533255, 0.6946945949961153, 0.6884818791691059, 0.6826464924874314, 0.674695217464828, 0.6690870522688999, 0.6604539573176332, 0.6535212533274407 ], "peak_layer": 1, "peak_normalized": 2.1746021240914017, "population_cv": 0.3960989175249597, "rest_mean": 0.00019674027215741161, "spike_contrast": 0.68956516628749, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013566523848567157, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004070796421729028, 0.0003637922345660627, 0.0003274149785283953, 0.0002957331307698041, 0.0002725222148001194, 0.000256980158155784, 0.0002415281196590513, 0.00022440051543526351, 0.00021120885503478348, 0.00019987139967270195, 0.00019166592392139137, 0.0001824002101784572, 0.00017469454905949533, 0.00016871507978066802, 0.00016249077452812344, 0.00015767784498166293, 0.00015292088210117072, 0.0001493355812272057, 0.00014505717263091356, 0.0001422565837856382, 0.00013998440408613533, 0.0001379159075440839, 0.00013545199180953205, 0.0001333056716248393, 0.00013166821736376733, 0.000130044951220043, 0.0001288819476030767, 0.00012778957898262888, 0.00012630112178158015, 0.00012525128840934485, 0.00012363519635982811, 0.00012233741290401667 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.264873631298542, "first_to_last_ratio": 2.3520309299879743, "imbalance_abs_log_ratio": 0.8552791804550044, "last_quartile_mean": 0.11261485889554024, "mean": 0.1660084358882159, "normalized": [ 2.1746021175238424, 1.943362568695452, 1.7490370652499811, 1.5797937888156908, 1.4558020518807187, 1.3727770906872316, 1.2902329816871188, 1.1987379486864336, 1.128268732118104, 1.067704514212167, 1.0238712059346726, 0.9743741284397964, 0.9332107750795899, 0.9012687970282692, 0.8680188148399602, 0.8423083473020997, 0.8168968752199287, 0.7977442908994873, 0.7748892254753884, 0.7599285915530729, 0.7477907763310051, 0.7367409150028685, 0.723578786768912, 0.7121132849551643, 0.7033659848486875, 0.694694622945607, 0.6884818276985868, 0.6826464794318051, 0.6746951840389442, 0.6690870220520635, 0.6604539434158253, 0.6535212611815249 ], "peak_layer": 1, "peak_normalized": 2.1746021175238424, "population_cv": 0.3960989241805914, "rest_mean": 0.17447124007675383, "spike_contrast": 0.6895651868880185, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12030929327011108, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.3610022962093353, 0.3226145803928375, 0.2903549075126648, 0.2622590959072113, 0.24167542159557343, 0.22789257764816284, 0.2141895592212677, 0.19900061190128326, 0.1873021274805069, 0.17724795639514923, 0.1699712574481964, 0.1617543250322342, 0.15492086112499237, 0.1496182233095169, 0.1440984457731247, 0.13983029127120972, 0.13561177253723145, 0.13243228197097778, 0.1286381483078003, 0.1261545568704605, 0.12413957715034485, 0.12230520695447922, 0.12012018263339996, 0.11821681261062622, 0.11676468700170517, 0.11532516777515411, 0.11429379135370255, 0.1133250743150711, 0.1120050922036171, 0.11107409000396729, 0.1096409261226654, 0.10849004238843918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010120303977600997, "first_to_last_ratio": 2.7021675066537045, "imbalance_abs_log_ratio": 0.994054231196686, "last_quartile_mean": 3.7452541164384456e-05, "mean": 6.280763545873924e-05, "normalized": [ 1.9944816746100333, 1.8472480968114133, 1.7373223217437128, 1.6248488809553971, 1.5471371990089653, 1.475823179261819, 1.3807460832170038, 1.2829322622274897, 1.184046410564159, 1.1043973349513228, 1.0901910170788929, 1.0456150919711364, 0.9888694060739084, 0.9664075020831666, 0.9198699716804313, 0.9041232034205934, 0.8692950074065211, 0.8446819175937094, 0.7923092109368705, 0.7644112750281646, 0.7503465778061283, 0.730017034806627, 0.7090250315296005, 0.6754099655367306, 0.6572034579968898, 0.6267819521510305, 0.6185439744898635, 0.6068927943478243, 0.594580659914726, 0.5858446061910916, 0.5555293303136664, 0.5250675682911103 ], "peak_layer": 1, "peak_normalized": 1.9944816746100333, "population_cv": 0.40483316550850257, "rest_mean": 6.624576713807915e-05, "spike_contrast": 0.6678422833882899, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4241724390303715e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012526867794804275, 0.0001160212850663811, 0.0001091171070584096, 0.00010205291619058698, 9.717202920001e-05, 9.269296424463391e-05, 8.67213966557756e-05, 8.057794184423983e-05, 7.43671553209424e-05, 6.936458521522582e-05, 6.847231998108327e-05, 6.567261152667925e-05, 6.210854917299002e-05, 6.0697770095430315e-05, 5.777485785074532e-05, 5.6785840570228174e-05, 5.4598363931290805e-05, 5.305247395881452e-05, 4.9763068091124296e-05, 4.801086470251903e-05, 4.712749432655983e-05, 4.585064380080439e-05, 4.453218571143225e-05, 4.242090290063061e-05, 4.1277395212091506e-05, 3.936669236281887e-05, 3.8849284464959055e-05, 3.811750138993375e-05, 3.734420533874072e-05, 3.679551446111873e-05, 3.4891483664978296e-05, 3.297825242043473e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021292950441420544, "first_to_last_ratio": 1.6819067487661699, "imbalance_abs_log_ratio": 0.5199281193278776, "last_quartile_mean": 0.0001266000654140953, "mean": 0.0001612693722563563, "normalized": [ 1.5193752902631465, 1.4431842082185322, 1.3854135927323608, 1.3303138916593422, 1.2819858609596173, 1.2449680699964556, 1.2025089225516594, 1.1549255897117912, 1.1161361523436344, 1.0820631347545688, 1.0617857510438193, 1.0341003696475617, 1.0077971864601107, 0.9864740856164548, 0.9634556759892069, 0.9447446572547388, 0.9240188125766456, 0.9078710595764491, 0.88634376010722, 0.8724890232241534, 0.8607832896405347, 0.849628883322604, 0.8359853832855236, 0.8234683594489096, 0.8139294954165563, 0.8040629029670215, 0.7969416676465471, 0.7902390259734583, 0.7809695092222964, 0.7743360767166025, 0.7639958481267083, 0.7557044635457686 ], "peak_layer": 1, "peak_normalized": 1.5193752902631465, "population_cv": 0.21463654613558777, "rest_mean": 0.00016614451305940747, "spike_contrast": 0.8122062500591205, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013494361191987992, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024502869928255677, 0.00023274141130968928, 0.00022342478041537106, 0.0002145388862118125, 0.00020674505503848195, 0.00020077521912753582, 0.00019392785907257348, 0.00018625412485562265, 0.00017999857664108276, 0.00017450364248361439, 0.00017123352154158056, 0.00016676871746312827, 0.0001625268196221441, 0.00015908805653452873, 0.0001553758920636028, 0.0001523583778180182, 0.0001490159338572994, 0.000146411795867607, 0.00014294010179582983, 0.0001407057570759207, 0.00013881798076909035, 0.00013701911666430533, 0.0001348188379779458, 0.0001328002254012972, 0.00013126189878676087, 0.0001296707196161151, 0.00012852228246629238, 0.00012744135165121406, 0.0001259464625036344, 0.00012487669300753623, 0.00012320913083385676, 0.00012187198444735259 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016320452232321259, "first_to_last_ratio": 1.2889611434881496, "imbalance_abs_log_ratio": 0.25383657880769345, "last_quartile_mean": 0.0001266171002498595, "mean": 0.0001437355554116948, "normalized": [ 1.188749445631, 1.1713423911172207, 1.1549430883999143, 1.1417765084687417, 1.1273283201397843, 1.1145504039766105, 1.0995991496765185, 1.0853099095412984, 1.0687879028680622, 1.0554743211679045, 1.0473286818228558, 1.0363809984280803, 1.0249947395076546, 1.015899258153915, 1.005968438235508, 0.9970397006238613, 0.9860146691908593, 0.977410712347053, 0.964005406402611, 0.9549550784847994, 0.9474810711889571, 0.940343993567483, 0.9294782140508817, 0.9176125795270506, 0.909513915953909, 0.8990879268536105, 0.8931112722873075, 0.8859642730588357, 0.8770245001904262, 0.8704731011044752, 0.8603223859779169, 0.851727642054894 ], "peak_layer": 1, "peak_normalized": 1.188749445631, "population_cv": 0.0978236443065259, "rest_mean": 0.0001456284474072702, "spike_contrast": 0.9168122095142401, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001335139386355877, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017086556181311607, 0.00016836354916449636, 0.00016600638628005981, 0.00016411388060078025, 0.0001620371622266248, 0.00016020052134990692, 0.0001580514945089817, 0.00015599762264173478, 0.00015362282283604145, 0.0001517091877758503, 0.00015053836978040636, 0.0001489647984271869, 0.00014732818817719817, 0.0001460208441130817, 0.00014459343219641596, 0.0001433100551366806, 0.00014172536612022668, 0.00014048867160454392, 0.00013856185250915587, 0.00013726099859923124, 0.0001361867180094123, 0.00013516086619347334, 0.00013359906733967364, 0.0001318935537710786, 0.0001307294878643006, 0.00012923090253025293, 0.00012837184476666152, 0.0001273445668630302, 0.00012605960364453495, 0.0001251179346581921, 0.00012365891598165035, 0.00012242354569025338 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0004412600719660986, "first_to_last_ratio": 3.098863205201153, "imbalance_abs_log_ratio": 1.131035336237652, "last_quartile_mean": 0.00014239417578210123, "mean": 0.0002548141455918085, "normalized": [ 2.1785577639630773, 1.9956115105027457, 1.9014578504944815, 1.7829105267225334, 1.6475854563109211, 1.565091756282257, 1.45799103716316, 1.3243443696030393, 1.2370613678820481, 1.1609137324491365, 1.1300355744023234, 1.0677196003316773, 1.0048909110245325, 0.948494062464312, 0.8853730946340664, 0.8440976906413151, 0.7906677717627395, 0.747069444517614, 0.6991826523616375, 0.6712814178889162, 0.6509454457824981, 0.6314362826751919, 0.6113540983289655, 0.5954000197507868, 0.5831757760496624, 0.5733601746441271, 0.5653129211221045, 0.559900286080882, 0.5532114024202266, 0.5496528914136888, 0.545014697375157, 0.5408984129541747 ], "peak_layer": 1, "peak_normalized": 2.1785577639630773, "population_cv": 0.4822008017525282, "rest_mean": 0.00027300682214209465, "spike_contrast": 0.573515676244785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015657369222026317, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005551273352466524, 0.0005085100419819355, 0.000484518357552588, 0.00045431082253344357, 0.00041982808033935726, 0.0003988075186498463, 0.0003715167404152453, 0.0003374616790097207, 0.000315220735501498, 0.0002958172408398241, 0.0002879490493796766, 0.0002720700576901436, 0.0002560604189056903, 0.00024168970412574708, 0.00022560558863915503, 0.00021508803183678538, 0.0002014733327087015, 0.0001903638622025028, 0.0001781616301741451, 0.0001710520009510219, 0.00016587010759394616, 0.0001608988968655467, 0.00015578167221974581, 0.00015171634731814265, 0.00014860143710393459, 0.00014610028301831335, 0.0001440497289877385, 0.0001426705130143091, 0.00014096609083935618, 0.0001400593318976462, 0.0001388774544466287, 0.00013782856694888324 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027412420604377985, 0.002683233004063368, 0.002711885841563344, 0.0027249811682850122, 0.0027012599166482687, 0.002700963756069541, 0.0026683746837079525, 0.002738022478297353, 0.0027370047755539417, 0.002654933836311102, 0.002663377905264497, 0.002684312639757991, 0.0027356327045708895, 0.002652956871315837, 0.002679797587916255, 0.002698024269193411, 0.0026484597474336624, 0.0026680538430809975, 0.002654345240443945, 0.002591830212622881, 0.002620960585772991, 0.002651234855875373, 0.0026270023081451654, 0.002615687670186162, 0.00258000404573977, 0.002626793459057808, 0.002573782578110695, 0.002592637902125716, 0.002581534907221794, 0.002551236655563116, 0.002563327783718705, 0.002589041367173195 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024784023116808385, "first_to_last_ratio": 4.996872150813687, "imbalance_abs_log_ratio": 1.6088121468463858, "last_quartile_mean": 4.959907391821616e-05, "mean": 0.00012175017718618619, "normalized": [ 2.94187592163941, 2.5796687009574604, 2.293010133276413, 2.014581184592976, 1.8111575385173901, 1.6842185280489494, 1.5559062012549134, 1.4047478191310576, 1.2749623745204015, 1.1699150302730006, 1.0919472369387608, 1.0017565099771182, 0.923449750141626, 0.8652102016053241, 0.794546741948017, 0.743905553038253, 0.6942834150416333, 0.6562793060663384, 0.6082995898528493, 0.5744438165059796, 0.5512557468449423, 0.5265581646918154, 0.5020472770644135, 0.47690127545745264, 0.4572061982209032, 0.43893553328289897, 0.42961039869683454, 0.41603121061550513, 0.3999104909112459, 0.3911954601706125, 0.3717234123738856, 0.35445927834161867 ], "peak_layer": 1, "peak_normalized": 2.94187592163941, "population_cv": 0.6874421460933116, "rest_mean": 0.0001329603533290706, "spike_contrast": 0.46040210094136547, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.121522601461038e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000358173914719373, 0.0003140751214232296, 0.00027917439001612365, 0.0002452756161801517, 0.00022050875122658908, 0.0002050539042102173, 0.00018943185568787158, 0.0001710282958811149, 0.00015522689500357956, 0.00014243736222852021, 0.00013294476957526058, 0.00012196403258712962, 0.00011243017070228234, 0.0001053394953487441, 9.673620661487803e-05, 9.057063289219514e-05, 8.45291287987493e-05, 7.990212179720402e-05, 7.406058284686878e-05, 6.993863644311205e-05, 6.711548485327512e-05, 6.410854985006154e-05, 6.112434493843466e-05, 5.806281478726305e-05, 5.566493564401753e-05, 5.3440478950506076e-05, 5.23051421623677e-05, 5.0651873607421294e-05, 4.8689173127058893e-05, 4.762811659020372e-05, 4.525739132077433e-05, 4.315547994337976e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002960815872938838, "first_to_last_ratio": 2.3434901925615312, "imbalance_abs_log_ratio": 0.8516413534183918, "last_quartile_mean": 0.00012634214908757713, "mean": 0.00018583460405352525, "normalized": [ 2.1691817718266027, 1.9356770461635995, 1.7512361881653136, 1.5771501673188, 1.4517248962140685, 1.3691574500653627, 1.2898282592750647, 1.2020702391870204, 1.1274711846152252, 1.06796916249687, 1.023464646242978, 0.9751293351351744, 0.9332415616222968, 0.9026335726010739, 0.8668647658197504, 0.8407791679025106, 0.8171800119835181, 0.7975248027052753, 0.7770671945621679, 0.7612080123915693, 0.7492445405290613, 0.7365475788483447, 0.7249228575947372, 0.7138182430319103, 0.7048361054118131, 0.6962681268232633, 0.6911654119184109, 0.6833348384348117, 0.6760742523950837, 0.6702828385523486, 0.6624247014500871, 0.654521068715886 ], "peak_layer": 1, "peak_normalized": 2.1691817718266027, "population_cv": 0.3943796352106793, "rest_mean": 0.00019526833030222742, "spike_contrast": 0.6908057343541223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013489248231053353, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004031090356875211, 0.0003597157774493098, 0.0003254402836319059, 0.0002930890768766403, 0.00026978072128258646, 0.00025443683261983097, 0.00023969472385942936, 0.00022338624694384634, 0.00020952316117472947, 0.0001984656264539808, 0.0001901951472973451, 0.00018121277389582247, 0.0001734285760903731, 0.00016774055256973952, 0.0001610934705240652, 0.0001562458637636155, 0.00015186032396741211, 0.00014820770593360066, 0.00014440597442444414, 0.00014145878958515823, 0.00013923556252848357, 0.0001368760276818648, 0.00013471575221046805, 0.00013265213056001812, 0.00013098293857183307, 0.00012939071166329086, 0.00012844245065934956, 0.00012698725913651288, 0.00012563799100462347, 0.0001245617459062487, 0.00012310143210925162, 0.00012163266364950687 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.26256811060011387, "first_to_last_ratio": 2.3434901455089974, "imbalance_abs_log_ratio": 0.8516413333404181, "last_quartile_mean": 0.11204148270189762, "mean": 0.16479998547583818, "normalized": [ 2.1691817568110046, 1.935676923537455, 1.7512361570514487, 1.5771502517930829, 1.4517248703109182, 1.369157420355369, 1.2898282606548555, 1.2020702692325518, 1.1274712340116186, 1.0679692103147351, 1.0234646460792782, 0.975129361875517, 0.9332415430463463, 0.9026335787242323, 0.8668647305888977, 0.8407791967519535, 0.8171800231493583, 0.7975247749486774, 0.7770672329681078, 0.7612079835803747, 0.7492445111174014, 0.7365475542475244, 0.7249228843083022, 0.7138182179223472, 0.7048361082067401, 0.6962681206044983, 0.6911654218524023, 0.6833348067031066, 0.676074287607254, 0.6702828621875807, 0.6624247142471237, 0.6545210852099367 ], "peak_layer": 1, "peak_normalized": 2.1691817568110046, "population_cv": 0.3943796257753467, "rest_mean": 0.17316590856622766, "spike_contrast": 0.6908057237027376, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11962400078773498, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.35748112201690674, 0.3189995288848877, 0.28860369324684143, 0.2599143385887146, 0.2392442375421524, 0.22563712298870087, 0.2125636786222458, 0.19810116291046143, 0.1858072429895401, 0.17600131034851074, 0.16866695880889893, 0.16070130467414856, 0.1537981927394867, 0.14875400066375732, 0.1428592950105667, 0.13856039941310883, 0.13467125594615936, 0.13143207132816315, 0.12806066870689392, 0.12544706463813782, 0.12347548454999924, 0.12138302624225616, 0.11946728080511093, 0.11763723194599152, 0.11615698039531708, 0.11474497616291046, 0.11390405148267746, 0.11261356621980667, 0.11141703277826309, 0.11046260595321655, 0.10916758328676224, 0.10786506533622742 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.0001010398200378404, "first_to_last_ratio": 2.701478580108855, "imbalance_abs_log_ratio": 0.9937992453824708, "last_quartile_mean": 3.740167358046165e-05, "mean": 6.270599271829269e-05, "normalized": [ 1.9907220064146427, 1.8444741506775382, 1.742884685905627, 1.6239253294262157, 1.5436270879110234, 1.4787029723792133, 1.3841644782908762, 1.2821103017907618, 1.1824374683113494, 1.1042100658007115, 1.0908679078034196, 1.0498354286471192, 0.9920033594422748, 0.9660018389944104, 0.9202838356691797, 0.8989347700163286, 0.8686518679586676, 0.8410583182307008, 0.7938538755561205, 0.7653997098760733, 0.7511903220519177, 0.7265360595007238, 0.7106247592487483, 0.6758121071782397, 0.6556340454607258, 0.6278169583683687, 0.6205082786340235, 0.6073860029650574, 0.5911469097961847, 0.5867782134359579, 0.5577710930141501, 0.5246457912436486 ], "peak_layer": 1, "peak_normalized": 1.9907220064146427, "population_cv": 0.40468494452628273, "rest_mean": 6.614368087159366e-05, "spike_contrast": 0.6673725457789733, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4142476690467446e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001248301996383816, 0.00011565958266146481, 0.0001092893144232221, 0.00010182984988205135, 9.679466893430799e-05, 9.272353781852871e-05, 8.679540769662708e-05, 8.039599924813956e-05, 7.414591527776793e-05, 6.92405883455649e-05, 6.84039550833404e-05, 6.583097274415195e-05, 6.220455543370917e-05, 6.0574104281840846e-05, 5.770731149823405e-05, 5.636859714286402e-05, 5.446967770694755e-05, 5.273939677863382e-05, 4.9779395340010524e-05, 4.799514863407239e-05, 4.710413486463949e-05, 4.5558164856629446e-05, 4.456043097889051e-05, 4.237746907165274e-05, 4.111218368052505e-05, 3.936788561986759e-05, 3.890958760166541e-05, 3.808674227911979e-05, 3.7068453821120784e-05, 3.679451037896797e-05, 3.497559009701945e-05, 3.2898435165407136e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021103879953443538, "first_to_last_ratio": 1.6754332028061545, "imbalance_abs_log_ratio": 0.5160717603794217, "last_quartile_mean": 0.00012596073611348402, "mean": 0.0001601102544555033, "normalized": [ 1.5150372860566723, 1.439510593902249, 1.3841055212355642, 1.3272119947749268, 1.2794362361966427, 1.2428436489609036, 1.2018556832108258, 1.1546727794959446, 1.1147996317367452, 1.0814138049096294, 1.0614905005088058, 1.0346901524019803, 1.0074919017055521, 0.9875011592687566, 0.9623166883068708, 0.9431047552100263, 0.9243559814545742, 0.9073018837351503, 0.8887802361951885, 0.87382889642758, 0.8622859040817829, 0.849391060863955, 0.8374811583413132, 0.8253926631762389, 0.8155827011133436, 0.8058320881575772, 0.7999867009830005, 0.7910419816457831, 0.7825317922950641, 0.7756782882808636, 0.7662211489344684, 0.7568251764320209 ], "peak_layer": 1, "peak_normalized": 1.5150372860566723, "population_cv": 0.21304026877727222, "rest_mean": 0.00016491277050658095, "spike_contrast": 0.8136220583009953, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013417666777968407, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024257300538010895, 0.00023048040748108178, 0.0002216094871982932, 0.00021250025019980967, 0.0002048508613370359, 0.00019899201288353652, 0.0001924294192576781, 0.00018487495253793895, 0.00017849085270427167, 0.00017314543947577477, 0.00016995551413856447, 0.00016566450358368456, 0.00016130978474393487, 0.00015810906188562512, 0.00015407676983159035, 0.00015100074233487248, 0.00014799887139815837, 0.0001452683354727924, 0.00014230282977223396, 0.00013990896695759147, 0.00013806081551592797, 0.00013599621888715774, 0.00013408932136371732, 0.00013215382932685316, 0.00013058315380476415, 0.00012902198068331927, 0.00012808607425540686, 0.0001266539329662919, 0.00012529136438388377, 0.00012419404811225832, 0.00012267986312508583, 0.00012117547157686204 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016164849512279034, "first_to_last_ratio": 1.282986844747637, "imbalance_abs_log_ratio": 0.24919083207168544, "last_quartile_mean": 0.00012599388355738483, "mean": 0.00014267631695474847, "normalized": [ 1.1849632230478688, 1.1695181917643926, 1.1537907432295644, 1.136648454640294, 1.1262413432306055, 1.112705815659591, 1.0977204646042988, 1.082200264843305, 1.06813080649915, 1.0535985061590996, 1.046252801833764, 1.0365714691025223, 1.024670983408149, 1.0169605538784676, 1.0046472025165352, 0.995758555548206, 0.9858673222379531, 0.9760103584099877, 0.9653106304982666, 0.9567951741123486, 0.9498417329681972, 0.9401737612553472, 0.9304537733640251, 0.9205681496416466, 0.911480005330001, 0.9022600866995591, 0.8969925816929462, 0.8873573495747816, 0.8789659160771423, 0.8714788501373707, 0.8627683840784355, 0.8532965439561776 ], "peak_layer": 1, "peak_normalized": 1.1849632230478688, "population_cv": 0.09608600673281108, "rest_mean": 0.00014451252193086676, "spike_contrast": 0.918680321330362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013276081008370966, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016906618839129806, 0.00016686254821252078, 0.00016461861378047615, 0.00016217281518038362, 0.00016068796685431153, 0.00015875676763243973, 0.0001566187129355967, 0.00015440434799529612, 0.00015239696949720383, 0.0001503235544078052, 0.00014927549636922777, 0.00014789419947192073, 0.00014619628200307488, 0.0001450961863156408, 0.00014333936269395053, 0.00014207116328179836, 0.00014065991854295135, 0.00013925356324762106, 0.00013772696547675878, 0.0001365120115224272, 0.00013551992014981806, 0.00013414052955340594, 0.00013275371748022735, 0.0001313432730967179, 0.00013004661013837904, 0.00012873114610556513, 0.0001279795978916809, 0.00012660487846005708, 0.0001254076196346432, 0.00012433939264155924, 0.00012309661542531103, 0.00012174520816188306 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00043634913890855387, "first_to_last_ratio": 3.0823770886184128, "imbalance_abs_log_ratio": 1.125701081333854, "last_quartile_mean": 0.00014156254292174708, "mean": 0.0002526328457861382, "normalized": [ 2.161262991989335, 1.9867412804887739, 1.8904526133838582, 1.7788259407438674, 1.6404609794852394, 1.5723911781894129, 1.459249713339863, 1.3282687448916313, 1.2383315143134188, 1.1700416770361293, 1.131363959578303, 1.0685850444189446, 1.0069989316428685, 0.9451411723514572, 0.8866050347634621, 0.8407661253267313, 0.7895843736134068, 0.7479926194272515, 0.7047300318080335, 0.6729704299307414, 0.6547401608613147, 0.6315820650248338, 0.6127131155987415, 0.5974089218754778, 0.5839699651029305, 0.5735248801032955, 0.5690230706448541, 0.5599717580649735, 0.5552403508520525, 0.5513694456095706, 0.5470291494736481, 0.5426627600655782 ], "peak_layer": 1, "peak_normalized": 2.161262991989335, "population_cv": 0.4790543297146424, "rest_mean": 0.0002705939723118381, "spike_contrast": 0.5751893185853856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015564276254735886, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005460060201585293, 0.0005019161035306752, 0.0004775904235430062, 0.00044938985956832767, 0.0004144343256484717, 0.0003972376580350101, 0.000368654407793656, 0.00033556431299075484, 0.00031284321448765695, 0.000295590958558023, 0.0002858196967281401, 0.0002699596807360649, 0.0002544010058045387, 0.00023877370404079556, 0.0002239855530206114, 0.00021240513888187706, 0.00019947494729422033, 0.0001889675040729344, 0.0001780379534466192, 0.00017001443484332412, 0.00016540887008886784, 0.00015955837443470955, 0.00015479145804420114, 0.00015092511603143066, 0.00014752999413758516, 0.00014489122258964926, 0.00014375391765497625, 0.00014146725879982114, 0.00014027194993104786, 0.00013929403212387115, 0.00013819753075949848, 0.00013709443737752736 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0023513585329055786, 0.004408807028084993, 0.005783648230135441, 0.006751363631337881, 0.002628475893288851, 0.004723479505628347, 0.006024772301316261, 0.007445912808179855, 0.003561305580660701, 0.005543973762542009, 0.006855612155050039, 0.007876631803810596, 0.0035210708156228065, 0.0058924350887537, 0.007390674669295549, 0.008790239691734314, 0.004162859171628952, 0.006355749908834696, 0.007784162648022175, 0.00937180407345295, 0.004737259820103645, 0.006999304983764887, 0.008728929795324802, 0.010217384435236454, 0.005580419674515724, 0.008008211851119995, 0.010209310799837112, 0.012040933594107628, 0.005235649179667234, 0.007745372597128153, 0.010180563665926456, 0.011906592175364494 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024914481400628574, "first_to_last_ratio": 5.005367240493075, "imbalance_abs_log_ratio": 1.6105107847992817, "last_quartile_mean": 4.977553135177004e-05, "mean": 0.0001223312358433759, "normalized": [ 2.9462138255793877, 2.588435651553335, 2.2854731426364707, 2.0157214492628985, 1.8168708763555168, 1.6875649797632875, 1.5554268829788211, 1.3974214212146046, 1.275680469442624, 1.1684152736282016, 1.0925056707858531, 1.0000365768115411, 0.9220243844233116, 0.8633153808206192, 0.7954940059538663, 0.747131175135411, 0.6955592946052511, 0.6567700236831091, 0.6067846463225012, 0.5734077695641915, 0.5496986650820618, 0.5277889932643854, 0.50090777460584, 0.47622023530363405, 0.457550957846772, 0.43796965221258716, 0.42825768814458814, 0.4157111150450942, 0.4005890169587395, 0.39033503431894295, 0.37039430607646523, 0.3543236606200867 ], "peak_layer": 1, "peak_normalized": 2.9462138255793877, "population_cv": 0.6883836085455175, "rest_mean": 0.00013360307773971862, "spike_contrast": 0.46004396487681354, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.14632896031253e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036041397834196687, 0.00031664653215557337, 0.0002795847540255636, 0.0002465856960043311, 0.0002222600596724078, 0.0002064419095404446, 0.00019027729285880923, 0.00017094828945118934, 0.00015605556836817414, 0.00014293368440121412, 0.00013364756887312979, 0.00012233571032993495, 0.00011279238242423162, 0.00010561043745838106, 9.73137648543343e-05, 9.139747999142855e-05, 8.508862811140716e-05, 8.0343488662038e-05, 7.422871567541733e-05, 7.014568109298125e-05, 6.724531704094261e-05, 6.456507981056347e-05, 6.127666711108759e-05, 5.8256609918316826e-05, 5.597277413471602e-05, 5.357736881705932e-05, 5.238929225015454e-05, 5.0854454457294196e-05, 4.9004549509845674e-05, 4.775016714120284e-05, 4.531079321168363e-05, 4.334485129220411e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00029868123783671763, "first_to_last_ratio": 2.352020548935129, "imbalance_abs_log_ratio": 0.8552747667903616, "last_quartile_mean": 0.00012698921273113228, "mean": 0.0001871975605354237, "normalized": [ 2.1745956556501205, 1.943389916860048, 1.7490167059616495, 1.579801534641211, 1.4557872843351107, 1.3727714128797337, 1.290211227528517, 1.198748061313539, 1.1282731192017241, 1.0677025479901412, 1.0238711728275764, 0.9743826669000581, 0.9332092274716193, 0.901273964826617, 0.8680163418407979, 0.8423074677807364, 0.8168973316425937, 0.7977426572048737, 0.7748904888155362, 0.7599280177062464, 0.7477886704872306, 0.7367391764207782, 0.7235805599766263, 0.7121146362854748, 0.7033711688788506, 0.6946931548498627, 0.6884857337520279, 0.6826539310873735, 0.6746941940291191, 0.6690858034871798, 0.660454275348722, 0.6535218920183048 ], "peak_layer": 1, "peak_normalized": 2.1745956556501205, "population_cv": 0.39609817473730136, "rest_mean": 0.00019674051630621156, "spike_contrast": 0.6895661448911525, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013566559937316924, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0004070790018886328, 0.00036379785160534084, 0.00032741166069172323, 0.00029573499341495335, 0.00027251982828602195, 0.00025697945966385305, 0.00024152439436875284, 0.000224402712774463, 0.00021120997553225607, 0.00019987131236121058, 0.00019166618585586548, 0.0001824020582716912, 0.00017469449085183442, 0.00016871628758963197, 0.00016249054169747978, 0.00015767790318932384, 0.00015292118769139051, 0.0001493354793637991, 0.0001450576091883704, 0.00014225667109712958, 0.00013998421491123736, 0.00013791577657684684, 0.00013545251567848027, 0.00013330612273421139, 0.00013166936696507037, 0.00013004486390855163, 0.00012888284982182086, 0.00012779115058947355, 0.00012630110722966492, 0.00012525123020168394, 0.00012363542919047177, 0.00012233770394232124 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.2648735139518976, "first_to_last_ratio": 2.3520205903150884, "imbalance_abs_log_ratio": 0.855274784383728, "last_quartile_mean": 0.1126153040677309, "mean": 0.1660086705815047, "normalized": [ 2.174595632268317, 1.943389981095013, 1.7490168198184282, 1.5798016086666014, 1.4557873374041121, 1.3727712901980273, 1.2902112306135503, 1.1987480127171795, 1.1282731510481394, 1.0677025559476454, 1.0238711946273023, 0.9743826246747078, 0.9332091864770874, 0.9012739856841682, 0.8680163310272434, 0.8423074257801569, 0.8168973360433722, 0.7977426245274992, 0.7748904637775177, 0.7599280557788093, 0.7477886868929706, 0.7367391104705062, 0.7235805912981054, 0.7121146568868953, 0.7033711391232357, 0.6946931471398464, 0.6884857463566807, 0.682653951912613, 0.6746941853139277, 0.6690857619715078, 0.6604542663637077, 0.6535219080951254 ], "peak_layer": 1, "peak_normalized": 2.174595632268317, "population_cv": 0.39609818748801673, "rest_mean": 0.1744714594549603, "spike_contrast": 0.6895661390160055, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12030961066484451, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.36100172996520996, 0.32261958718299866, 0.2903519570827484, 0.262260764837265, 0.2416733205318451, 0.2278919368982315, 0.21418625116348267, 0.19900256395339966, 0.18730312585830688, 0.17724788188934326, 0.1699714958667755, 0.16175596415996552, 0.1549208164215088, 0.14961929619312286, 0.14409823715686798, 0.1398303359746933, 0.13561204075813293, 0.13243219256401062, 0.12863853573799133, 0.12615464627742767, 0.12413940578699112, 0.12230508029460907, 0.12012065201997757, 0.11821720749139786, 0.11676570773124695, 0.11532508581876755, 0.11429460346698761, 0.11332647502422333, 0.1120050847530365, 0.1110740378499031, 0.10964113473892212, 0.10849030315876007 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010120356273546349, "first_to_last_ratio": 2.7021644745482813, "imbalance_abs_log_ratio": 0.9940531090948486, "last_quartile_mean": 3.7452776723512216e-05, "mean": 6.280785908074904e-05, "normalized": [ 1.9944873163456722, 1.8472744197374709, 1.7373237819064158, 1.6248552595134786, 1.5471235814302835, 1.4758134067763151, 1.3807453375978929, 1.2829373095670376, 1.18406084586495, 1.104394561286083, 1.0901943179184503, 1.04561090576924, 0.9888677388041794, 0.9664019181449638, 0.9198712145053942, 0.9041143079769479, 0.8692998477179782, 0.8446721332610243, 0.7923141515826202, 0.7644051939108963, 0.7503405467650113, 0.7300062685920449, 0.7090277780404898, 0.6754004942706008, 0.6572075474610624, 0.6267776932652476, 0.618543683654507, 0.6069002486680706, 0.5945805123210514, 0.585842809953234, 0.5555319861870183, 0.5250728812043677 ], "peak_layer": 1, "peak_normalized": 1.9944873163456722, "population_cv": 0.404835319326031, "rest_mean": 6.624604820562699e-05, "spike_contrast": 0.6678381428743124, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.424163780640811e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001252694783033803, 0.00011602335143834352, 0.00010911758727161214, 0.00010205368016613647, 9.717151988297701e-05, 9.269268048228696e-05, 8.672165859024972e-05, 8.057854574872181e-05, 7.436832675011829e-05, 6.936465797480196e-05, 6.847277109045535e-05, 6.567258242284879e-05, 6.210866558831185e-05, 6.069763549021445e-05, 5.7775141613092273e-05, 5.678548404830508e-05, 5.459886233438738e-05, 5.305204831529409e-05, 4.9763555580284446e-05, 4.801065369974822e-05, 4.7127283323789015e-05, 4.585013084579259e-05, 4.4532516767503694e-05, 4.242045906721614e-05, 4.127779902773909e-05, 3.936656503356062e-05, 3.884940451825969e-05, 3.811810529441573e-05, 3.734432903002016e-05, 3.679553265101276e-05, 3.489177470328286e-05, 3.297870352980681e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0002129299391526729, "first_to_last_ratio": 1.6819035009332732, "imbalance_abs_log_ratio": 0.5199261882838202, "last_quartile_mean": 0.00012660056836466538, "mean": 0.0001612697526525153, "normalized": [ 1.5193801883639617, 1.4431946098051458, 1.385412670946279, 1.330312377973009, 1.2819779644638467, 1.2449644115522398, 1.2025034693572292, 1.15492638462487, 1.1161420015875916, 1.082061394531008, 1.0617850512168836, 1.0341029835254725, 1.0077924632434987, 0.9864749169321705, 0.9634524108641764, 0.9447429702319522, 0.9240217763435827, 0.9078672036949645, 0.8863448276060427, 0.8724873261680857, 0.860780266694715, 0.8496270597212868, 0.835986840267667, 0.8234693045553775, 0.8139348844586144, 0.8040606454444867, 0.7969452920911189, 0.7902469974302787, 0.780967396404848, 0.7743340697791917, 0.7639952190801171, 0.7557046210402876 ], "peak_layer": 1, "peak_normalized": 1.5193801883639617, "population_cv": 0.21463676643173402, "rest_mean": 0.00016614488736700473, "spike_contrast": 0.8122069076744362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001349440251942724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024503006716258824, 0.00023274363775271922, 0.000223425158765167, 0.0002145391481462866, 0.00020674426923505962, 0.000200775102712214, 0.00019392743706703186, 0.00018625469238031656, 0.00017999994452111423, 0.00017450377345085144, 0.00017123381257988513, 0.00016676953237038106, 0.00016252644127234817, 0.00015908856585156173, 0.0001553757319925353, 0.00015235846512950957, 0.0001490167633164674, 0.00014641151938121766, 0.00014294061111286283, 0.00014070581528358161, 0.00013881782069802284, 0.00013701914576813579, 0.00013481939095072448, 0.00013280069106258452, 0.00013126307749189436, 0.00012967066140845418, 0.0001285231701331213, 0.00012744293780997396, 0.0001259464188478887, 0.00012487666390370578, 0.00012320932000875473, 0.00012187229731353 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016320142822223715, "first_to_last_ratio": 1.2889451135618213, "imbalance_abs_log_ratio": 0.25382414241573026, "last_quartile_mean": 0.0001266162744286703, "mean": 0.00014373415069712792, "normalized": [ 1.1884743462891088, 1.1713432082551107, 1.1549462763142666, 1.141829986160868, 1.1273728485863819, 1.1144091299418613, 1.0997826147068885, 1.0853573683237072, 1.068690221812727, 1.0554384700276902, 1.0473609881004726, 1.0364070219594264, 1.025017513262844, 1.015761575885501, 1.006046000385858, 0.9971413723222552, 0.9860728003762654, 0.9774913363845525, 0.9641287247440273, 0.9550391277708558, 0.9473133601248869, 0.9403037775179377, 0.9293980025104189, 0.9176260020019316, 0.9094819029195437, 0.8991813518536786, 0.8931890476284186, 0.8860794380468366, 0.8769637206693581, 0.8703930215998912, 0.8602546600212342, 0.8517047834951949 ], "peak_layer": 1, "peak_normalized": 1.1884743462891088, "population_cv": 0.09781347741247146, "rest_mean": 0.00014562865706264146, "spike_contrast": 0.9167413819240867, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013350381632335485, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017082435078918934, 0.00016836202121339738, 0.00016600522212684155, 0.0001641199633013457, 0.00016204197891056538, 0.00016017864982131869, 0.0001580763200763613, 0.0001560029195388779, 0.00015360728139057755, 0.00015170255210250616, 0.00015054154209792614, 0.00014896708307787776, 0.00014733002171851695, 0.00014599962742067873, 0.00014460316742770374, 0.00014332326827570796, 0.00014173233648762107, 0.00014049888704903424, 0.00013857822341378778, 0.00013727173791266978, 0.0001361612812615931, 0.0001351537648588419, 0.00013358623255044222, 0.00013189419405534863, 0.00013072360889054835, 0.00012924306793138385, 0.00012838176917284727, 0.00012735987547785044, 0.00012604963558260351, 0.00012510520173236728, 0.00012364797294139862, 0.00012241906370036304 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00044128536683274433, "first_to_last_ratio": 3.0989416798064484, "imbalance_abs_log_ratio": 1.1310606595922121, "last_quartile_mean": 0.0001423987323505571, "mean": 0.0002548205006860371, "normalized": [ 2.1788974669107217, 1.9948922239742408, 1.9013861017158324, 1.7832746019711707, 1.6474456862757656, 1.56528080717771, 1.458232784418858, 1.3245892215128081, 1.2376166576486232, 1.160533346247024, 1.1299807802432356, 1.0675072615016112, 1.0048234765069712, 0.9484981041710848, 0.8855756138701967, 0.8439855543529297, 0.7905268728467926, 0.7470509271970999, 0.6989608308202429, 0.6713105901225432, 0.6509228727076454, 0.631461080612247, 0.611226351598172, 0.5954626643405772, 0.5833123929329473, 0.5733046444152264, 0.5652547933675338, 0.559916360487464, 0.5532517426009346, 0.5496220513891801, 0.5450027610740823, 0.5408933749885276 ], "peak_layer": 1, "peak_normalized": 2.1788974669107217, "population_cv": 0.4822174837001359, "rest_mean": 0.0002730129350244102, "spike_contrast": 0.5735309033794403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015658135525882245, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0005552277434617281, 0.0005083394353277981, 0.0004845121584367007, 0.0004544149269349873, 0.00041980293462984264, 0.0003988656389992684, 0.00037158760824240744, 0.0003375324886292219, 0.00031537009635940194, 0.0002957276883535087, 0.00028794226818718016, 0.00027202273486182094, 0.00025604962138459086, 0.00024169676180463284, 0.0002256628213217482, 0.00021506482153199613, 0.0002014424535445869, 0.00019036389130633324, 0.00017810954886954278, 0.00017106370069086552, 0.0001658684923313558, 0.00016090922872535884, 0.00015575300494674593, 0.00015173609426710755, 0.00014863995602354407, 0.00014608977653551847, 0.00014403850946109742, 0.00014267816732171923, 0.00014097988605499268, 0.00014005496632307768, 0.0001388778764521703, 0.00013783072063233703 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0036361459642648697, 0.005152894649654627, 0.006386642809957266, 0.007284422870725393, 0.0037505170330405235, 0.0054016136564314365, 0.006585344672203064, 0.007922882214188576, 0.0044892593286931515, 0.006135277915745974, 0.007316684816032648, 0.0082859443500638, 0.0044554853811860085, 0.006464940495789051, 0.007893112488090992, 0.009196000173687935, 0.004961193539202213, 0.006907128728926182, 0.008217990398406982, 0.009676996618509293, 0.005381465889513493, 0.007502692751586437, 0.0091153085231781, 0.010543142445385456, 0.006136673036962748, 0.008469858206808567, 0.010564134456217289, 0.012301022186875343, 0.005773921962827444, 0.008156762458384037, 0.010457510128617287, 0.012218860909342766 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024784039669611957, "first_to_last_ratio": 4.996847129630424, "imbalance_abs_log_ratio": 1.6088071394647374, "last_quartile_mean": 4.959935540682636e-05, "mean": 0.0001217503483985638, "normalized": [ 2.9419062071052267, 2.5796112881191693, 2.2930860326581057, 2.0145554032379356, 1.8111535572920587, 1.6841756414503508, 1.5558994713918, 1.4047664015764878, 1.2749699043555138, 1.1699192416770665, 1.0919281315608147, 1.001760718810264, 0.9234449853785138, 0.8652188455092179, 0.7945394094365259, 0.7439156225148, 0.6942882953063699, 0.6562804150522287, 0.6082998698912725, 0.574440737760942, 0.551250788347098, 0.5265559301830911, 0.5020487822241618, 0.47689842352430634, 0.4572026867293029, 0.43893712719402944, 0.4296155316374904, 0.41603615348749273, 0.3999129464790325, 0.391194073390914, 0.3717285968379238, 0.3544587798804922 ], "peak_layer": 1, "peak_normalized": 2.9419062071052267, "population_cv": 0.6874423943830519, "rest_mean": 0.00013296059464833147, "spike_contrast": 0.4603997057303065, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.121501864981837e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00035817810567095876, 0.0003140685730613768, 0.00027918402338400483, 0.00024527282221242785, 0.00022050857660360634, 0.00020504897111095488, 0.0001894313027150929, 0.00017103079881053418, 0.00015522803005296737, 0.0001424380752723664, 0.000132942630443722, 0.00012196471652714536, 0.00011242974869674072, 0.00010534069588175043, 9.673544991528615e-05, 9.057198622031137e-05, 8.452984184259549e-05, 7.99023691797629e-05, 7.406072109006345e-05, 6.99383599567227e-05, 6.711497553624213e-05, 6.410836795112118e-05, 6.112461414886639e-05, 5.806254921481013e-05, 5.566458639805205e-05, 5.34407481609378e-05, 5.230584065429866e-05, 5.0652546633500606e-05, 4.868954056291841e-05, 4.762801472679712e-05, 4.5258086174726486e-05, 4.315547994337976e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00029608183103846386, "first_to_last_ratio": 2.343484699021421, "imbalance_abs_log_ratio": 0.8516390092453431, "last_quartile_mean": 0.0001263425492652459, "mean": 0.0001858347804954974, "normalized": [ 2.16920774571012, 1.9356424765570717, 1.751272112160408, 1.5771369240348272, 1.451727119925344, 1.369130935687747, 1.289826251583108, 1.2020808437231323, 1.127478492836466, 1.067970341065492, 1.0234512239081746, 0.9751296621832141, 0.9332398141881068, 0.9026389800429313, 0.8668613586828855, 0.8407822065969367, 0.8171794710239251, 0.7975274126338607, 0.777066300159633, 0.7612061150730458, 0.7492407752334779, 0.7365489154759011, 0.7249215428669998, 0.7138135717041164, 0.7048334785593375, 0.6962720074755282, 0.6911692191098191, 0.683337870004401, 0.6760758813563923, 0.6702804011180051, 0.662430102042749, 0.6545204472768429 ], "peak_layer": 1, "peak_normalized": 2.16920774571012, "population_cv": 0.39438035124793563, "rest_mean": 0.0001952685916977417, "spike_contrast": 0.6908033638721538, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001348922000033781, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00040311424527317286, 0.00035970969474874437, 0.00032544726855121553, 0.00029308689408935606, 0.00026978139067068696, 0.00025443214690312743, 0.00023969457834027708, 0.0002233884297311306, 0.0002095247182296589, 0.0001984660339076072, 0.0001901928335428238, 0.00018121300672646612, 0.0001734284160193056, 0.0001677417167229578, 0.00016109299031086266, 0.00015624657680746168, 0.0001518603676231578, 0.00014820833166595548, 0.00014440594532061368, 0.0001414585713064298, 0.00013923499500378966, 0.00013687640603166074, 0.00013471563579514623, 0.00013265138841234148, 0.00013098257477395236, 0.0001293915556743741, 0.00012844328011851758, 0.0001269879430765286, 0.0001256384130101651, 0.00012456141121219844, 0.0001231025526067242, 0.00012163266364950687 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.2625683266669512, "first_to_last_ratio": 2.343484691137257, "imbalance_abs_log_ratio": 0.8516390058810522, "last_quartile_mean": 0.1120418356731534, "mean": 0.16480014030821621, "normalized": [ 2.169207748902955, 1.9356423730461585, 1.7512721262865893, 1.5771370154883375, 1.4517271231757105, 1.3691309069442503, 1.2898263254837699, 1.202080803995401, 1.1274784933366124, 1.0679703770106845, 1.0234512066143264, 0.9751297116000902, 0.9332398524746174, 0.9026389696372329, 0.8668612939874407, 0.8407822044491968, 0.8171795266543764, 0.7975273711861923, 0.7770662316421936, 0.7612060929585753, 0.7492407781338722, 0.7365489418998509, 0.7249215250846839, 0.7138135688166896, 0.704833501980704, 0.6962720326387369, 0.6911692482611179, 0.6833378266924969, 0.6760758677045711, 0.6702804240542598, 0.6624300595814382, 0.6545204702768674 ], "peak_layer": 1, "peak_normalized": 2.169207748902955, "population_cv": 0.3943803506949354, "rest_mean": 0.17316613815448903, "spike_contrast": 0.6908033707584418, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11962375193834304, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.35748574137687683, 0.3189941346645355, 0.28860989212989807, 0.2599124014377594, 0.23924483358860016, 0.22563296556472778, 0.21256355941295624, 0.19810308516025543, 0.18580861389636993, 0.1760016679763794, 0.16866490244865417, 0.16070151329040527, 0.15379805862903595, 0.1487550288438797, 0.1428588628768921, 0.13856102526187897, 0.13467130064964294, 0.13143262267112732, 0.12806062400341034, 0.1254468709230423, 0.12347498536109924, 0.12138336896896362, 0.11946716904640198, 0.11763657629489899, 0.11615666002035141, 0.11474572867155075, 0.11390478909015656, 0.11261416971683502, 0.11141739785671234, 0.11046230792999268, 0.10916856676340103, 0.10786506533622742 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 0.00010104048214998329, "first_to_last_ratio": 2.7014816335786884, "imbalance_abs_log_ratio": 0.9938003756776099, "last_quartile_mean": 3.740187639778014e-05, "mean": 6.270627807225537e-05, "normalized": [ 1.9907250147127677, 1.8444603036108247, 1.7429056467158477, 1.6239117898045463, 1.5436487234069576, 1.4786997242989472, 1.3841625886805662, 1.282123032540977, 1.1824417181452251, 1.1042125830371328, 1.090862943651666, 1.0498276343781445, 0.9920003536086259, 0.9660066676365643, 0.9202805760437535, 0.8989382213875909, 0.8686500616317681, 0.8410510099026279, 0.7938571089189717, 0.7653969230084752, 0.751185801344563, 0.7265337395686793, 0.7106236720469997, 0.6758027080339929, 0.6556269427545545, 0.6278192648368973, 0.6205119527339269, 0.6073894466988616, 0.5911483388457922, 0.586775311152339, 0.5577767930941091, 0.5246434037673043 ], "peak_layer": 1, "peak_normalized": 1.9907250147127677, "population_cv": 0.4046861823965828, "rest_mean": 6.614403860617636e-05, "spike_contrast": 0.6673673413246971, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.414237118908204e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012483095633797348, 0.00011565924069145694, 0.00010929112613666803, 0.0001018294642562978, 9.67964660958387e-05, 9.272375609725714e-05, 8.679568418301642e-05, 8.039716340135783e-05, 7.41465191822499e-05, 6.924106128280982e-05, 6.84039550833404e-05, 6.583078356925398e-05, 6.220465002115816e-05, 6.0574682720471174e-05, 5.770736970589496e-05, 5.636907008010894e-05, 5.446981231216341e-05, 5.273917849990539e-05, 4.977982462150976e-05, 4.799519228981808e-05, 4.7104065743042156e-05, 4.555822670226917e-05, 4.456056558410637e-05, 4.237707253196277e-05, 4.1111925384029746e-05, 3.9368209399981424e-05, 3.89099950552918e-05, 3.808713154285215e-05, 3.706871211761609e-05, 3.679449582705274e-05, 3.4976106690010056e-05, 3.2898435165407136e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00021103927974763792, "first_to_last_ratio": 1.675431813331652, "imbalance_abs_log_ratio": 0.5160709310565479, "last_quartile_mean": 0.00012596112719620578, "mean": 0.00016011051866371417, "normalized": [ 1.5150472374771589, 1.439502583502474, 1.3841101446264577, 1.327205260326266, 1.2794452130965426, 1.2428385988073838, 1.2018548814818522, 1.154676418187547, 1.1148050630726085, 1.0814152014364051, 1.0614838409967562, 1.034687172582471, 1.0074911480481314, 0.9875049829338368, 0.9623163727426111, 0.9431041078017948, 0.9243534563643369, 0.9073040220081667, 0.888778951337658, 0.8738270000353379, 0.8622815727966864, 0.8493922949455803, 0.8374788674952537, 0.8253865750371285, 0.8155793557629833, 0.8058361207192702, 0.7999903796437624, 0.7910448570865577, 0.7825332275919701, 0.7756749178925374, 0.766226246614208, 0.7568239275482656 ], "peak_layer": 1, "peak_normalized": 1.5150472374771589, "population_cv": 0.21304076077436968, "rest_mean": 0.0001649131305317429, "spike_contrast": 0.8136187466924137, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001341764145763591, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024257499899249524, 0.00023047950526233763, 0.00022161059314385056, 0.00021249952260404825, 0.00020485263667069376, 0.00019899153267033398, 0.00019242960843257606, 0.00018487584020476788, 0.00017849201685748994, 0.00017314594879280776, 0.00016995472833514214, 0.00016566429985687137, 0.00016130993026308715, 0.00015810993500053883, 0.00015407697355840355, 0.00015100088785402477, 0.00014799871132709086, 0.00014526891754940152, 0.00014230285887606442, 0.00013990889419801533, 0.00013806034985464066, 0.00013599664089269936, 0.00013408917584456503, 0.00013215307262726128, 0.00013058283366262913, 0.00012902283924631774, 0.00012808687461074442, 0.0001266546023543924, 0.00012529180094134063, 0.00012419371341820806, 0.00012268088175915182, 0.00012117547157686204 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00016164389126060996, "first_to_last_ratio": 1.2829777917904304, "imbalance_abs_log_ratio": 0.24918377588932214, "last_quartile_mean": 0.00012599118417710997, "mean": 0.0001426773092134681, "normalized": [ 1.1847160153617624, 1.1696203114063919, 1.15376099486318, 1.1365907777467705, 1.126248503505431, 1.112765799827437, 1.097614510342432, 1.0821504120145113, 1.06828860482523, 1.0536786878013262, 1.046275103221197, 1.0365937358276223, 1.0246881313142864, 1.0168969779036778, 1.0047039605050023, 0.9958410772863212, 0.9859208451030141, 0.9762312163738422, 0.965237112554715, 0.9568250330809598, 0.9498426746244831, 0.9400597234006037, 0.9304691287067991, 0.9205814319068928, 0.9114724424808572, 0.9022957305012854, 0.8969570718532309, 0.8872712168205663, 0.8789155388108255, 0.8714969614364647, 0.8626996589435726, 0.8532906096493081 ], "peak_layer": 1, "peak_normalized": 1.1847160153617624, "population_cv": 0.0960837578275494, "rest_mean": 0.00014451401323269778, "spike_contrast": 0.9186590596986468, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013275910750962794, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000169032093253918, 0.00016687827883288264, 0.0001646155142225325, 0.00016216571384575218, 0.0001606901059858501, 0.00015876643010415137, 0.00015660468488931656, 0.00015439830895047635, 0.00015242054359987378, 0.00015033603995107114, 0.000149279716424644, 0.00014789840497542173, 0.00014619974535889924, 0.0001450881245546043, 0.00014334845764096826, 0.00014208392531145364, 0.00014066853327676654, 0.00013928604312241077, 0.0001377174339722842, 0.00013651722110807896, 0.00013552099699154496, 0.00013412519183475524, 0.00013275683159008622, 0.00013134608161635697, 0.0001300464355153963, 0.000128737126942724, 0.00012797542149201035, 0.00012659346975851804, 0.00012540130410343409, 0.00012434284144546837, 0.00012308766599744558, 0.00012174520816188306 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00043632716551655903, "first_to_last_ratio": 3.082278345441195, "imbalance_abs_log_ratio": 1.1256690460714511, "last_quartile_mean": 0.00014155994904285762, "mean": 0.00025262987765017897, "normalized": [ 2.1611427674158366, 1.9871567751170711, 1.8899978819847916, 1.7784604477508343, 1.6406551319992353, 1.5724690971097588, 1.4592296472822097, 1.3280082080145534, 1.2379389344592089, 1.1701308820499392, 1.131654892203279, 1.0685236385646504, 1.0072825277339317, 0.9454673581714634, 0.8865158004683366, 0.8408935109532261, 0.7895809204213243, 0.7481431077843942, 0.7047226439788088, 0.6727720648712864, 0.6548656488865329, 0.6315662143568483, 0.6127430670138975, 0.5973169234484098, 0.5839093169225268, 0.5733930286828336, 0.5691911560855093, 0.5599482690481828, 0.5552538441515218, 0.5513421690146183, 0.547054988271426, 0.542669135783553 ], "peak_layer": 1, "peak_normalized": 2.1611427674158366, "population_cv": 0.47902914554115894, "rest_mean": 0.0002705909153317205, "spike_contrast": 0.5751866206559582, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001556402741698548, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.000545969232916832, 0.0005020151729695499, 0.0004774699336849153, 0.0004492922453209758, 0.00041447850526310503, 0.0003972526756115258, 0.00036864500725641847, 0.00033549455110915005, 0.00031274036155082285, 0.00029561002156697214, 0.00028588983695954084, 0.0002699409960769117, 0.00025446966174058616, 0.00023885330301709473, 0.00022396037820726633, 0.00021243482478894293, 0.00019947173132095486, 0.00018900330178439617, 0.0001780339953256771, 0.00016996232443489134, 0.00016543862875550985, 0.0001595524954609573, 0.00015479720605071634, 0.0001509001012891531, 0.00014751293929293752, 0.0001448562106816098, 0.00014379469212144613, 0.00014145966270007193, 0.0001402737107127905, 0.00013928550470154732, 0.00013820243475493044, 0.00013709443737752736 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.027978340163826942, 0.0140878576785326, 0.014187770895659924, 0.014337530359625816, 0.014477464370429516, 0.009757626801729202, 0.009825476445257664, 0.00987118761986494, 0.009999874979257584, 0.007595349568873644, 0.007658981718122959, 0.007689184043556452, 0.007755559403449297, 0.00625597033649683, 0.006315375678241253, 0.00635211868211627, 0.006408242508769035, 0.005408597644418478, 0.005475745070725679, 0.005517482291907072, 0.005585856735706329, 0.004830927588045597, 0.004899685736745596, 0.004948681220412254, 0.004989070352166891, 0.004419817589223385, 0.004463871940970421, 0.004542885348200798, 0.004592607729136944, 0.004137600306421518, 0.004193917382508516, 0.004230021964758635 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00024312340610777028, "first_to_last_ratio": 2.286272500967691, "imbalance_abs_log_ratio": 0.8269227625411849, "last_quartile_mean": 0.00010634051977831405, "mean": 0.00017363850702167838, "normalized": [ 1.061084599304074, 1.6344608012282997, 1.6065574901355, 1.6039206242202213, 1.1359733847422597, 1.548756457352017, 1.2967221414405534, 1.313883475593909, 1.1957449434685963, 1.344067980756563, 1.165161355052429, 1.1171979507437393, 1.0391889074947966, 1.1329181602378293, 1.0774500297071945, 1.0521010358096583, 0.862761425176938, 0.9019434894037968, 0.9280163978632505, 0.8870730707939211, 0.8354471012965897, 0.8049103603266032, 0.8255366455754076, 0.7297241480789424, 0.6373588767497196, 0.6752584188072907, 0.6298229328646214, 0.6250051046248064, 0.6032481952321226, 0.5674884668477911, 0.6733729976615692, 0.4878430314089892 ], "peak_layer": 2, "peak_normalized": 1.6344608012282997, "population_cv": 0.3218023265359089, "rest_mean": 0.00018114369622371332, "spike_contrast": 0.7348336602687936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001331104853306897, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.00018424514564685524, 0.0002838053333107382, 0.000278960244031623, 0.00027850238257087767, 0.00019724872254300863, 0.00026892375899478793, 0.00022516089666169137, 0.0002281407651025802, 0.0002076273667626083, 0.00023338195751421154, 0.00020231687813065946, 0.00019398858421482146, 0.0001804432104108855, 0.00019671821792144328, 0.00018708681454882026, 0.00018268525309395045, 0.000149808605783619, 0.00015661212091799825, 0.0001611393818166107, 0.00015403004363179207, 0.00014506578736472875, 0.00013976343325339258, 0.00014334495062939823, 0.0001267082116100937, 0.00011067004379583523, 0.00011725086369551718, 0.00010936151375062764, 0.00010852495324797928, 0.00010474711598362774, 9.853785013547167e-05, 0.000116923481982667, 8.470833563478664e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00025287891548941843, "first_to_last_ratio": 2.2739168136310908, "imbalance_abs_log_ratio": 0.8215038131425536, "last_quartile_mean": 0.0001112085165004828, "mean": 0.0001807239029858465, "normalized": [ 1.0592123382591336, 1.645736468094544, 1.60181949161791, 1.5891655894821897, 1.1402836456293184, 1.5483130487587196, 1.300612463539172, 1.3089009659072275, 1.1885078058073624, 1.3408206015264876, 1.166937022076678, 1.1148751106540256, 1.0356001272424655, 1.1351884917697168, 1.0748875173492451, 1.0458945475116934, 0.8650051516349683, 0.9033374117584585, 0.9274894298403746, 0.8851829368231718, 0.8352656795149113, 0.8098299287299446, 0.8239713600606818, 0.7303600865627311, 0.6419462275110265, 0.6820467100618508, 0.635541252420758, 0.625618271589476, 0.6029839777576883, 0.5727133544881818, 0.6681570652306708, 0.4937959207892163 ], "peak_layer": 2, "peak_normalized": 1.645736468094544, "population_cv": 0.3200470231318611, "rest_mean": 0.00018847914530110925, "spike_contrast": 0.7366629038009043, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001388455944834277, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001914249878609553, 0.00029742391780018806, 0.00028948707040399313, 0.0002872002078220248, 0.00020607651094906032, 0.0002798171772155911, 0.00023505176068283617, 0.00023654969118069857, 0.00021479176939465106, 0.0002423183323116973, 0.00021089341316837817, 0.00020148458133917302, 0.00018715769692789763, 0.00020515569485723972, 0.0001942578674061224, 0.0001890181447379291, 0.00015632710710633546, 0.00016325466276612133, 0.00016761950973886997, 0.0001599737151991576, 0.00015095247363206, 0.0001463556254748255, 0.00014891132013872266, 0.00013199352542869747, 0.0001160150277428329, 0.0001232621434610337, 0.00011485749564599246, 0.00011306417582090944, 0.00010897361789830029, 0.0001035029927152209, 0.00012075195263605565, 8.924072608351707e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.22425555251538754, "first_to_last_ratio": 2.2739168285843645, "imbalance_abs_log_ratio": 0.8215038197185521, "last_quartile_mean": 0.09862082451581955, "mean": 0.1602677651681006, "normalized": [ 1.0592124025748084, 1.645736496509649, 1.6018195299533093, 1.5891655927499475, 1.1402836784663197, 1.5483129508886688, 1.300612441604666, 1.3089009387606192, 1.1885077887054891, 1.3408205692725423, 1.1669370180912673, 1.1148751103389594, 1.0356000505037584, 1.1351884646301031, 1.0748875233039092, 1.0458946120331756, 0.8650051766933494, 0.9033374073089909, 0.9274894888798537, 0.8851828780404403, 0.835265755835364, 0.8098299455472033, 0.8239713233255873, 0.730360099613448, 0.6419462245963619, 0.6820467317825786, 0.6355412485786012, 0.6256182682629655, 0.602983984145969, 0.5727133411523797, 0.6681570418174577, 0.49379591603225714 ], "peak_layer": 2, "peak_normalized": 1.645736496509649, "population_cv": 0.3200470194234096, "rest_mean": 0.16714519124340127, "spike_contrast": 0.7366629183018027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1231296643614769, "top_five_layers": [ 2, 3, 4, 6, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.16975760459899902, 0.26375851035118103, 0.25672003626823425, 0.254692018032074, 0.18275071680545807, 0.24814465641975403, 0.20844624936580658, 0.20977462828159332, 0.19047948718070984, 0.21489031612873077, 0.1870223879814148, 0.17867854237556458, 0.16597330570220947, 0.18193411827087402, 0.1722698211669922, 0.16762319207191467, 0.13863244652748108, 0.1447758674621582, 0.14864666759967804, 0.1418662816286087, 0.13386617600917816, 0.12978963553905487, 0.13205604255199432, 0.11705318093299866, 0.10288328677415848, 0.1093101054430008, 0.1018567755818367, 0.10026644170284271, 0.09663889557123184, 0.09178748726844788, 0.10708403587341309, 0.07913956791162491 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.143860511860112e-05, "first_to_last_ratio": 0.9636580872932836, "imbalance_abs_log_ratio": 0.037018728516889636, "last_quartile_mean": 7.413273033307632e-05, "mean": 7.686168260079285e-05, "normalized": [ 0.586022200160155, 0.9787081875553917, 0.912570839911241, 1.015732496910769, 0.8201270226617754, 1.1119332047208261, 1.0008210861718638, 1.0096344009845781, 0.9335569648148418, 1.090167719321714, 1.035494250546431, 0.9697380161483544, 1.0399314841024359, 1.0649519568255064, 1.06563466639597, 1.1389203959735625, 0.9952637999761301, 1.0027093290728673, 1.1427249020241628, 1.1090175842590573, 1.0432879501645442, 1.0450477353718262, 1.1125450117027875, 1.0594965280199307, 0.9180884623091344, 0.995296932026832, 0.9631737996002885, 0.9446177686211702, 0.9674620335911367, 0.9355074959711648, 1.1351899163905306, 0.8566258576930204 ], "peak_layer": 19, "peak_normalized": 1.1427249020241628, "population_cv": 0.10846075768175395, "rest_mean": 7.635363915512614e-05, "spike_contrast": 1.0425844542348655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.960511720739305e-05, "top_five_layers": [ 19, 16, 31, 23, 6 ], "top_five_spike_overlap": 1 }, "values": [ 4.504265234572813e-05, 7.522515807067975e-05, 7.014173024799675e-05, 7.807090878486633e-05, 6.303634290816262e-05, 8.546505705453455e-05, 7.692479266552255e-05, 7.760219887131825e-05, 7.175475911935791e-05, 8.379212522413582e-05, 7.958983042044565e-05, 7.453569560311735e-05, 7.993088365765288e-05, 8.185399929061532e-05, 8.190647349692881e-05, 8.753933798288926e-05, 7.64976502978243e-05, 7.706992619205266e-05, 8.78317587194033e-05, 8.52409575600177e-05, 8.018886728677899e-05, 8.032412733882666e-05, 8.551208156859502e-05, 8.143468585330993e-05, 7.056582398945466e-05, 7.650019688298926e-05, 7.403115887427703e-05, 7.260491111082956e-05, 7.436075975419953e-05, 7.190468022599816e-05, 8.725260704522952e-05, 6.584170478163287e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 8.107683152047684e-05, "first_to_last_ratio": 1.03534602173519, "imbalance_abs_log_ratio": 0.034735691361772646, "last_quartile_mean": 7.83089226388256e-05, "mean": 8.487996228723205e-05, "normalized": [ 0.5954853629443996, 1.025163681929567, 0.9520754208460104, 1.0126519094835282, 0.8443534832663374, 1.1548844288399933, 1.0326421147312754, 1.0242956755731092, 0.9710206925084107, 1.15116021353655, 1.072832454869962, 1.0193061391332177, 1.0419196505755806, 1.0947619162703996, 1.1089694127677372, 1.1239932261258834, 0.9835832115239759, 1.0364664516448652, 1.11612142168979, 1.093437276508762, 1.0179007506072324, 1.033774483284387, 1.0848380474290875, 1.0276879808812729, 0.8974590825111308, 0.9692041024722606, 0.9378977551023808, 0.9155031739616566, 0.918412101050971, 0.8874638084461439, 1.0536745962271137, 0.8010599732570078 ], "peak_layer": 6, "peak_normalized": 1.1548844288399933, "population_cv": 0.11200209045145568, "rest_mean": 8.468612051931107e-05, "spike_contrast": 1.0146492401244358, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.592670783400536e-05, "top_five_layers": [ 6, 10, 16, 19, 15 ], "top_five_spike_overlap": 0 }, "values": [ 5.054477514931932e-05, 8.701585466042161e-05, 8.081212581600994e-05, 8.59538558870554e-05, 7.166869181673974e-05, 9.802654676605016e-05, 8.76506237545982e-05, 8.694217831362039e-05, 8.242019976023585e-05, 9.771043551154435e-05, 9.106197830988094e-05, 8.651866664877161e-05, 8.843810064718127e-05, 9.292335016652942e-05, 9.41292819334194e-05, 9.540450264466926e-05, 8.348650590050966e-05, 8.797523332759738e-05, 9.473634418100119e-05, 9.281091479351744e-05, 8.639937732368708e-05, 8.774673915468156e-05, 9.20810125535354e-05, 8.723011706024408e-05, 7.617629307787865e-05, 8.226600766647607e-05, 7.960872608236969e-05, 7.770787487970665e-05, 7.795478450134397e-05, 7.532789459219202e-05, 8.943585999077186e-05, 6.799394031986594e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.1427921979011444e-05, "first_to_last_ratio": 0.40836765899304117, "imbalance_abs_log_ratio": 0.8955873853644011, "last_quartile_mean": 5.2472132665570825e-05, "mean": 3.864622581772892e-05, "normalized": [ 0.2626641640048693, 0.5716512870137274, 0.5157707114648397, 0.4843895386483896, 0.5109475889421058, 0.7512596968779521, 0.675728069412672, 0.6632970161544233, 0.7004288748821966, 0.9130902556041411, 0.8717836320781894, 0.8044820763577291, 0.9241814793896349, 0.9531622930600316, 1.0319070449118461, 1.0273708471578158, 0.9893551980130566, 1.0844069350216101, 1.1887300698220302, 1.2100415764612447, 1.1128447755732793, 1.2346303137099222, 1.3033621630444512, 1.3524689418392815, 1.2055635543985064, 1.318388624044801, 1.3615446320871405, 1.2933406942036905, 1.3753836688642536, 1.3573239762703455, 1.6442457326403273, 1.3062545680454958 ], "peak_layer": 31, "peak_normalized": 1.6442457326403273, "population_cv": 0.32680115045148705, "rest_mean": 3.691592014042221e-05, "spike_contrast": 1.2999777952883038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.798987647518516e-05, "top_five_layers": [ 31, 29, 27, 30, 24 ], "top_five_spike_overlap": 1 }, "values": [ 1.0150978596357163e-05, 2.2092164726927876e-05, 1.99325913854409e-05, 1.8719827494351193e-05, 1.974619590328075e-05, 2.9033351893303916e-05, 2.6114339561900124e-05, 2.5633926270529628e-05, 2.7068932467955165e-05, 3.528749221004546e-05, 3.369114710949361e-05, 3.1090195989236236e-05, 3.571612614905462e-05, 3.683612521854229e-05, 3.9879312680568546e-05, 3.970400575781241e-05, 3.823484439635649e-05, 4.190823528915644e-05, 4.5939930714666843e-05, 4.676354001276195e-05, 4.300725049688481e-05, 4.771380190504715e-05, 5.037002847529948e-05, 5.226782013778575e-05, 4.65904813609086e-05, 5.09507444803603e-05, 5.261856131255627e-05, 4.998273652745411e-05, 5.3153387852944434e-05, 5.24554488947615e-05, 6.354389188345522e-05, 5.048180901212618e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031520502852799837, "first_to_last_ratio": 1.31300997816315, "imbalance_abs_log_ratio": 0.27232219480674863, "last_quartile_mean": 0.00024006293460843153, "mean": 0.00030979480334281106, "normalized": [ 0.607123837738274, 1.0354820546705408, 0.9347765087792952, 1.1085716606000595, 0.9306423427143409, 1.2565833735558338, 1.1285140374487672, 1.138017377238765, 1.0370737734279494, 1.2142990636699202, 1.1645184658462944, 1.1206382096550715, 1.1232577862254296, 1.1815362250572234, 1.1924463988431009, 1.2309950595128432, 1.0374639291264394, 1.065463636492051, 1.1630951913863559, 1.1082935818936954, 1.0270641612904527, 0.9934767629460804, 1.0585185268550952, 0.9428720751183239, 0.8088208526771234, 0.8683362720246933, 0.8018903045940483, 0.8027332776454693, 0.7640102071384416, 0.7171383971343437, 0.8334550561316563, 0.6028915925620211 ], "peak_layer": 6, "peak_normalized": 1.2565833735558338, "population_cv": 0.17765693833110424, "rest_mean": 0.0003117367305094376, "spike_contrast": 0.9601319554288655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00029930839664302764, "top_five_layers": [ 6, 16, 10, 15, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00018808380991686136, 0.00032078695949167013, 0.00028958890470676124, 0.00034342973958700895, 0.0002883081615436822, 0.00038928299909457564, 0.00034960778430104256, 0.0003525518695823848, 0.00032128006569109857, 0.00037618353962898254, 0.00036076176911592484, 0.00034716789377853274, 0.0003479794249869883, 0.00036603378248400986, 0.0003694136976264417, 0.00038135587237775326, 0.0003214009338989854, 0.0003300750977359712, 0.0003603208460845053, 0.000343343592248857, 0.00031817913986742496, 0.00030777393840253353, 0.00032792353886179626, 0.00029209686908870935, 0.0002505684969946742, 0.0002690060646273196, 0.00024842144921422005, 0.0002486825978849083, 0.00023668639187235385, 0.00022216574870981276, 0.000258200045209378, 0.0001867726823547855 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014031016267836094, 0.014151837676763535, 0.014265215955674648, 0.014401593245565891, 0.009722241200506687, 0.009786579757928848, 0.009839320555329323, 0.00996283907443285, 0.007562503684312105, 0.007635751739144325, 0.007660485804080963, 0.007726665120571852, 0.006238248664885759, 0.006290202960371971, 0.006327399518340826, 0.006382675375789404, 0.005390914157032967, 0.0054517327807843685, 0.005509276874363422, 0.0055727302096784115, 0.0048178439028561115, 0.004884926602244377, 0.004938251338899136, 0.004975184798240662, 0.004412420094013214, 0.004455919377505779, 0.004530393984168768, 0.0045819394290447235, 0.004127359017729759, 0.004185030702501535, 0.004221425391733646, 0.004281661007553339 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.6193598993559135e-05, "first_to_last_ratio": 1.5723739147519853, "imbalance_abs_log_ratio": 0.4525865249799748, "last_quartile_mean": 3.573806361600873e-05, "mean": 4.4294749727669114e-05, "normalized": [ 1.5642840408451666, 1.493254721227763, 1.1816235069560812, 1.0717429494942774, 1.3553344243310417, 1.3257825682631335, 1.1255110934944355, 1.0314977817441078, 1.194075821513187, 1.0893775754580661, 1.0407197138445958, 0.9875719064328349, 1.135803276632048, 1.0324555952346264, 0.9635988909717662, 0.90208003019059, 0.9904506854292476, 0.9196323607404276, 0.8708711781381843, 0.8075366439875266, 0.9218591005239072, 0.8776357461813653, 0.8517186042296377, 0.8109904231766266, 0.8722342267309339, 0.8500552841464633, 0.7844009556175089, 0.7665260139039981, 0.8384393111065888, 0.8154485842192296, 0.7754690286134897, 0.7520179566211438 ], "peak_layer": 1, "peak_normalized": 1.5642840408451666, "population_cv": 0.20749976292329902, "rest_mean": 4.538663482957485e-05, "spike_contrast": 0.8460325450777163, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.839857017737813e-05, "top_five_layers": [ 1, 2, 5, 6, 9 ], "top_five_spike_overlap": 0 }, "values": [ 6.928957009222358e-05, 6.614334415644407e-05, 5.23397175129503e-05, 4.747258572024293e-05, 6.003419912303798e-05, 5.872520705452189e-05, 4.985423220205121e-05, 4.5689936087001115e-05, 5.289128966978751e-05, 4.8253707063850015e-05, 4.6098419261397794e-05, 4.374425043351948e-05, 5.03101218782831e-05, 4.573236219584942e-05, 4.2682371713453904e-05, 3.9957409171620384e-05, 4.3871765228686854e-05, 4.073488526046276e-05, 3.857502088067122e-05, 3.576963354134932e-05, 4.083351814188063e-05, 3.887465572915971e-05, 3.7726662412751466e-05, 3.592261782614514e-05, 3.863539677695371e-05, 3.7652986065950245e-05, 3.474484401522204e-05, 3.395307794562541e-05, 3.713845944730565e-05, 3.612009095377289e-05, 3.4349206543993205e-05, 3.3310447179246694e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 9.100913848669734e-05, "first_to_last_ratio": 0.8419197805282016, "imbalance_abs_log_ratio": 0.17207054181035333, "last_quartile_mean": 0.00010809716150106397, "mean": 9.889523016681778e-05, "normalized": [ 1.0744641570554045, 0.9868986027413778, 0.8415148680287817, 0.7657672453742986, 1.0045876840265406, 0.9858056114400382, 0.8710937611903306, 0.8319330217638002, 1.0050985706445066, 0.9363425314153265, 0.9111662095698898, 0.8822483642571924, 1.0547921529408322, 0.9937743096656301, 0.9653849347701057, 0.9236053864198286, 1.0608213360421703, 1.0216834776263237, 0.9948854733452297, 0.9444045194208119, 1.0902032759381237, 1.070297753084291, 1.0352646449837608, 1.0035839360241054, 1.109605122267389, 1.0988987221493531, 1.0612028824132491, 1.0392747076439328, 1.1393068813062424, 1.1142247321092786, 1.0935193303224786, 1.0883457940193757 ], "peak_layer": 29, "peak_normalized": 1.1393068813062424, "population_cv": 0.09089893356811851, "rest_mean": 9.776359500527313e-05, "spike_contrast": 1.0740814107080978, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010500606003915891, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.00010625938011799008, 9.759956446941942e-05, 8.322180656250566e-05, 7.573072798550129e-05, 9.934893023455516e-05, 9.749147284310311e-05, 8.614701800979674e-05, 8.227420767070726e-05, 9.939945448422804e-05, 9.259981015929952e-05, 9.010999201564118e-05, 8.725015504751354e-05, 0.00010431391274323687, 9.827953908825293e-05, 9.547196532366797e-05, 9.134016727330163e-05, 0.00010491017019376159, 0.00010103962267749012, 9.838942787609994e-05, 9.339710231870413e-05, 0.0001078159039025195, 0.00010584734263829887, 0.00010238273534923792, 9.924966434482485e-05, 0.00010973465396091342, 0.00010867584205698222, 0.00010494790330994874, 0.00010277931141899899, 0.00011267201625742018, 0.000110191511339508, 0.00010814384586410597, 0.0001076322078006342 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.08070781361311674, "first_to_last_ratio": 0.8419197775247833, "imbalance_abs_log_ratio": 0.1720705453776983, "last_quartile_mean": 0.09586164355278015, "mean": 0.0877012787386775, "normalized": [ 1.074464079725317, 0.986898606287062, 0.8415149217948403, 0.7657672391640048, 1.0045876578450608, 0.9858055872189405, 0.8710938052118494, 0.8319330371629138, 1.0050985716236776, 0.9363425240221704, 0.9111662153888755, 0.8822483595926541, 1.0547922001671663, 0.993774279010469, 0.9653849224854107, 0.9236053602662656, 1.060821305580311, 1.021683475130349, 0.9948854782496288, 0.9444045803369281, 1.0902032654624292, 1.07029776221103, 1.0352646575744078, 1.0035839255047951, 1.109605076031185, 1.0988987393251362, 1.0612029192639736, 1.0392747445534645, 1.1393068879878778, 1.1142247084644445, 1.0935193626414328, 1.0883457447159286 ], "peak_layer": 29, "peak_normalized": 1.1393068879878778, "population_cv": 0.09089892559784572, "rest_mean": 0.08669773351263117, "spike_contrast": 1.0740813996684329, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0931204229593277, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.09423187375068665, 0.08655226975679398, 0.0738019347190857, 0.0671587660908699, 0.08810362219810486, 0.08645641058683395, 0.07639604061841965, 0.07296159118413925, 0.08814842998981476, 0.0821184366941452, 0.07991044223308563, 0.07737430930137634, 0.09250662475824356, 0.08715527504682541, 0.08466549217700958, 0.08100137114524841, 0.09303538501262665, 0.08960294723510742, 0.08725272864103317, 0.08282548934221268, 0.09561222046613693, 0.09386648237705231, 0.09079403430223465, 0.0880155935883522, 0.0973137840628624, 0.09637482464313507, 0.09306885302066803, 0.09114572405815125, 0.0999186709523201, 0.09771893173456192, 0.09590304642915726, 0.09544931352138519 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0474835562490625e-05, "first_to_last_ratio": 0.595870196340598, "imbalance_abs_log_ratio": 0.5177324270116084, "last_quartile_mean": 1.7579056020622374e-05, "mean": 1.45315484587627e-05, "normalized": [ 0.6891108502390517, 0.67218760206668, 0.6248497232673742, 0.6113105238885437, 0.8536003812586072, 0.8434482348779837, 0.7260190066641083, 0.7461464268756399, 0.9515026546685825, 0.8968707547028398, 0.8666952712848448, 0.9173542983424958, 1.0474470628626384, 1.0570493146141757, 0.9856965747654381, 1.0403130781822671, 1.05710126231986, 1.0899464822844398, 1.0639055355381295, 0.9944043245776518, 1.1636416255463544, 1.1162669452398895, 1.1491653646620976, 1.158233556446643, 1.241320970071456, 1.2666472916328237, 1.1327368714456698, 1.1012802199127885, 1.2948691657915095, 1.2555094531837419, 1.2151558495322814, 1.1702133232533922 ], "peak_layer": 29, "peak_normalized": 1.2948691657915095, "population_cv": 0.19478066534993646, "rest_mean": 1.408557606434139e-05, "spike_contrast": 1.202634475953173, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6939799388637765e-05, "top_five_layers": [ 29, 26, 30, 25, 31 ], "top_five_spike_overlap": 1 }, "values": [ 1.0013847713707946e-05, 9.767926712811459e-06, 9.080034033104312e-06, 8.883288501237985e-06, 1.2404135304677766e-05, 1.2256608897587284e-05, 1.0550180377322249e-05, 1.0842662959476002e-05, 1.3826806934957858e-05, 1.3032920833211392e-05, 1.2594424333656207e-05, 1.3330578440218233e-05, 1.522102775197709e-05, 1.5360563338617794e-05, 1.4323697541840374e-05, 1.5117359907890204e-05, 1.5361318219220266e-05, 1.5838610124774277e-05, 1.546019484521821e-05, 1.445023463020334e-05, 1.690951467026025e-05, 1.622108720766846e-05, 1.669915218371898e-05, 1.6830927052069455e-05, 1.8038315829471685e-05, 1.8406346498522907e-05, 1.6460420738440007e-05, 1.600330688233953e-05, 1.8816454030456953e-05, 1.8244496459374204e-05, 1.7658096112427302e-05, 1.7005011613946408e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.836043712610262e-05, "first_to_last_ratio": 0.6391636816233681, "imbalance_abs_log_ratio": 0.44759470463089907, "last_quartile_mean": 0.00010695294349716278, "mean": 9.121693358338234e-05, "normalized": [ 0.7254188545798665, 0.6902023390007371, 0.6496225556724757, 0.6277900804246503, 0.8680656002782756, 0.8588952084862168, 0.7971788633739051, 0.7782428701770085, 0.9734232055545582, 0.9237343700472408, 0.9096320021673064, 0.8918620199754489, 1.0844263684244682, 1.0342037426828012, 1.0147200800673386, 0.9791745321037664, 1.1290928536156495, 1.0928814238713915, 1.0660995437899934, 1.013549283136708, 1.170635644938427, 1.1503062295054498, 1.1129730585678779, 1.0777737723208793, 1.1919439416858, 1.1807148042349604, 1.1393605783805036, 1.1153320412921235, 1.222157091120308, 1.1946826876057626, 1.1714041050576325, 1.1645002478604691 ], "peak_layer": 29, "peak_normalized": 1.222157091120308, "population_cv": 0.17156610618115575, "rest_mean": 8.883977756744975e-05, "spike_contrast": 1.1712498490939807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010405357606941834, "top_five_layers": [ 29, 30, 25, 26, 31 ], "top_five_spike_overlap": 1 }, "values": [ 6.617048347834498e-05, 6.295814091572538e-05, 5.925657751504332e-05, 5.7265086070401594e-05, 7.91822822066024e-05, 7.834578718757257e-05, 7.271621143445373e-05, 7.098892820067704e-05, 8.879267988959327e-05, 8.426021668128669e-05, 8.297384192701429e-05, 8.135291864164174e-05, 9.891804802464321e-05, 9.433689410798252e-05, 9.255965414922684e-05, 8.931729826144874e-05, 0.00010299238783773035, 9.968929225578904e-05, 9.724633127916604e-05, 9.245285764336586e-05, 0.00010678179387468845, 0.00010492740693734959, 0.00010152198956348002, 9.831121860770509e-05, 0.00010872547136386856, 0.00010770118387881666, 0.0001039289782056585, 0.00010173716873396188, 0.00011148142220918089, 0.00010897529136855155, 0.00010685189045034349, 0.00010622214176692069 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.421776268121903e-05, "first_to_last_ratio": 0.5089233562442377, "imbalance_abs_log_ratio": 0.6754578508927926, "last_quartile_mean": 0.00010653423942130757, "mean": 8.540443025140121e-05, "normalized": [ 0.5542697148172375, 0.5395320644633284, 0.526750599523528, 0.5251350203104778, 0.7537064875235374, 0.7453106043033457, 0.7183783488935762, 0.7156010196903468, 0.915261682615852, 0.8828592813937833, 0.8733821992497209, 0.8643716402564832, 1.0724322757212523, 1.0352702478756863, 1.026729960575756, 1.0038080492636434, 1.1695135624376918, 1.1411903421843295, 1.1203495535033974, 1.0697851231751558, 1.23599072599408, 1.2140634563521424, 1.1785251437606976, 1.1385123493568967, 1.2633504856351931, 1.2489710766708417, 1.2127527443953423, 1.1862530196716095, 1.298334269521613, 1.274402551981099, 1.2503128844914202, 1.2448935143909348 ], "peak_layer": 29, "peak_normalized": 1.298334269521613, "population_cv": 0.24541141609846925, "rest_mean": 8.214501078193145e-05, "spike_contrast": 1.2539446328637511, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010300529538653791, "top_five_layers": [ 29, 30, 25, 31, 26 ], "top_five_spike_overlap": 1 }, "values": [ 4.73370891995728e-05, 4.6078428567852825e-05, 4.498683483689092e-05, 4.4848857214674354e-05, 6.436987314373255e-05, 6.365282752085477e-05, 6.135269359219819e-05, 6.111549737397581e-05, 7.816740253474563e-05, 7.540009391959757e-05, 7.459070911863819e-05, 7.382116746157408e-05, 9.159046749118716e-05, 8.841666567604989e-05, 8.768728730501607e-05, 8.572965452913195e-05, 9.98816394712776e-05, 9.746271098265424e-05, 9.56828152993694e-05, 9.136438893619925e-05, 0.00010555908374954015, 0.00010368639777880162, 0.00010065126843983307, 9.723399853101e-05, 0.0001078957284335047, 0.00010666766320355237, 0.00010357445717090741, 0.00010131126327905804, 0.00011088349856436253, 0.00010883962386287749, 0.00010678225953597575, 0.00010631942132022232 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014064586594031425, "first_to_last_ratio": 1.0563608173139223, "imbalance_abs_log_ratio": 0.054829809986132017, "last_quartile_mean": 0.00013314188072399702, "mean": 0.0001347423383322166, "normalized": [ 1.1217990599984589, 1.0213028410040081, 0.9566177087867302, 0.90173442925003, 1.162761655685041, 1.2034553361266669, 1.0271109783027095, 0.9557256445604886, 1.16094674777681, 1.05596774404836, 1.0011734009390578, 0.9818696062747876, 1.1123781705815696, 1.0319357929888284, 0.9826152250905227, 0.9109733958837348, 1.0154471845455557, 0.955267408663159, 0.9085227032176484, 0.853750823711138, 0.9825495622515935, 0.9600374683174735, 0.9243229318330685, 0.9067574744309754, 0.9933509912573504, 0.986398830187635, 0.9577583846169437, 0.9375602577665346, 1.0292265528944584, 1.007559112024919, 0.9944595916547317, 0.9986629853290112 ], "peak_layer": 6, "peak_normalized": 1.2034553361266669, "population_cv": 0.07980954026492859, "rest_mean": 0.0001359050228428613, "spike_contrast": 0.9452471975467027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012846384197473526, "top_five_layers": [ 6, 5, 9, 1, 13 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001511538284830749, 0.0001376127329422161, 0.00012889690697193146, 0.00012150180555181578, 0.00015667322441004217, 0.0001621563860680908, 0.00013839533494319767, 0.00012877670815214515, 0.00015642867947462946, 0.0001422835630364716, 0.00013490044511854649, 0.00013229940668679774, 0.000149884435813874, 0.00013904544175602496, 0.00013239987310953438, 0.0001227466855198145, 0.00013682372809853405, 0.0001287149643758312, 0.0001224164734594524, 0.00011503638233989477, 0.00013239102554507554, 0.0001293576933676377, 0.0001245454332092777, 0.00012217862240504473, 0.00013384643534664065, 0.000132909684907645, 0.00012905060430057347, 0.00012632906145881861, 0.00013868039241060615, 0.00013576087076216936, 0.00013399581075645983, 0.0001345621858490631 ] } } } } } }, "step": 0 }, { "modes": { "learned": { "bits_per_byte": 5.118322357706281, "forward": { "activation_sha256": { "attention_branch_output": "18380f2ab4362bd0b1746896f121e69ee1546f1794cea323af41740b6024651d", "mlp_branch_output": "28d398d2ea80931917268ed413eca333d2f81168391c83a3adc104502478e55b", "post_attention_state": "1ff52033329506a86a2df82471984832a6abc7bac6dc80a52bb5d468ca4544c7", "post_mlp_state": "6cc11327437b0ddd855bc16b53a520d7ca542f4fd424dbb567b2c52308727f3b", "pre_attention_input": "a1eaf8973066bfbad01efbb61e56182f134640c86d8dd38e1601245f8a0a6387", "pre_mlp_input": "51c96475782d47da4b4460ab98a6f5efa352f41865c65bc22f8b46e27bda20aa" }, "logits_sha256": "994050c6c37c378cc5716dc8662ae2251d60efe8d70e351d49deded85b14b9c5", "loss_nats": 3.54775071144104, "mixer_summary_sha256": "be87753ea159fa25b47c87030938167aea73f40104b512c76f39174a23668c39" }, "loss_nats": 3.54775071144104, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.692298173904419, "group": 1, "latest_source_mass": 0.5198995471000671, "layer": 1, "max_source_mass": 0.5198995471000671, "mixer_index": 1, "normalized_entropy": 0.9987751423083904, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.48010045289993286, "median": 0.4801749289035797, "p05": 0.47152742743492126, "p95": 0.4886074662208557 }, { "label": "current_group_partial", "mean": 0.5198995471000671, "median": 0.5198251008987427, "p05": 0.5113925337791443, "p95": 0.5284726023674011 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6927915811538696, "group": 1, "latest_source_mass": 0.4869963526725769, "layer": 2, "max_source_mass": 0.5130037069320679, "mixer_index": 2, "normalized_entropy": 0.9994869785003115, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5130037069320679, "median": 0.512967050075531, "p05": 0.5081846714019775, "p95": 0.5176204442977905 }, { "label": "current_group_partial", "mean": 0.4869963526725769, "median": 0.487032949924469, "p05": 0.48237958550453186, "p95": 0.49181535840034485 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6928644776344299, "group": 1, "latest_source_mass": 0.4918276369571686, "layer": 2, "max_source_mass": 0.5081723928451538, "mixer_index": 3, "normalized_entropy": 0.9995921458913142, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5081723928451538, "median": 0.5079053044319153, "p05": 0.49492087960243225, "p95": 0.5215321183204651 }, { "label": "current_group_partial", "mean": 0.4918276369571686, "median": 0.4920946955680847, "p05": 0.4784679412841797, "p95": 0.5050790905952454 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.692646861076355, "group": 1, "latest_source_mass": 0.48500266671180725, "layer": 3, "max_source_mass": 0.5149973630905151, "mixer_index": 4, "normalized_entropy": 0.9992781915621641, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5149973630905151, "median": 0.5156484842300415, "p05": 0.5058209896087646, "p95": 0.5222364664077759 }, { "label": "current_group_partial", "mean": 0.48500266671180725, "median": 0.4843515157699585, "p05": 0.4777635335922241, "p95": 0.49417904019355774 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6925361156463623, "group": 1, "latest_source_mass": 0.4838656783103943, "layer": 3, "max_source_mass": 0.5161343812942505, "mixer_index": 5, "normalized_entropy": 0.9991184196795125, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5161343812942505, "median": 0.5157076120376587, "p05": 0.5059124827384949, "p95": 0.5276854634284973 }, { "label": "current_group_partial", "mean": 0.4838656783103943, "median": 0.4842924475669861, "p05": 0.47231459617614746, "p95": 0.49408748745918274 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6928029656410217, "group": 1, "latest_source_mass": 0.4892655909061432, "layer": 4, "max_source_mass": 0.5107344388961792, "mixer_index": 6, "normalized_entropy": 0.999503402843469, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5107344388961792, "median": 0.5119034051895142, "p05": 0.4968262016773224, "p95": 0.5211055874824524 }, { "label": "current_group_partial", "mean": 0.4892655909061432, "median": 0.48809656500816345, "p05": 0.4788943827152252, "p95": 0.5031737685203552 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6924350261688232, "group": 1, "latest_source_mass": 0.48286139965057373, "layer": 4, "max_source_mass": 0.5171386003494263, "mixer_index": 7, "normalized_entropy": 0.9989725783915809, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5171386003494263, "median": 0.5181499719619751, "p05": 0.5028476715087891, "p95": 0.5284117460250854 }, { "label": "current_group_partial", "mean": 0.48286139965057373, "median": 0.4818500280380249, "p05": 0.47158828377723694, "p95": 0.4971523582935333 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6929639577865601, "group": 2, "latest_source_mass": 0.49272024631500244, "layer": 5, "max_source_mass": 0.5072797536849976, "mixer_index": 8, "normalized_entropy": 0.9997356654134593, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5072797536849976, "median": 0.5079033970832825, "p05": 0.49702075123786926, "p95": 0.5173761248588562 }, { "label": "completed_group_1", "mean": 0.49272024631500244, "median": 0.49209660291671753, "p05": 0.4826239049434662, "p95": 0.5029792189598083 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0978727340698242, "group": 2, "latest_source_mass": 0.3277629613876343, "layer": 5, "max_source_mass": 0.34929555654525757, "mixer_index": 9, "normalized_entropy": 0.9993268283944082, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.34929555654525757, "median": 0.3491864800453186, "p05": 0.34007197618484497, "p95": 0.3589988350868225 }, { "label": "completed_group_1", "mean": 0.32294148206710815, "median": 0.3218739628791809, "p05": 0.31213808059692383, "p95": 0.3344322443008423 }, { "label": "current_group_partial", "mean": 0.3277629613876343, "median": 0.328006386756897, "p05": 0.3200114071369171, "p95": 0.3352474272251129 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098320722579956, "group": 2, "latest_source_mass": 0.3298703134059906, "layer": 6, "max_source_mass": 0.3432672321796417, "mixer_index": 10, "normalized_entropy": 0.9997346051094084, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3432672321796417, "median": 0.34342849254608154, "p05": 0.3353024423122406, "p95": 0.3504941165447235 }, { "label": "completed_group_1", "mean": 0.3268624544143677, "median": 0.3270367980003357, "p05": 0.3202572166919708, "p95": 0.3326030373573303 }, { "label": "current_group_partial", "mean": 0.3298703134059906, "median": 0.32962095737457275, "p05": 0.32568222284317017, "p95": 0.33469292521476746 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.098150610923767, "group": 2, "latest_source_mass": 0.32557153701782227, "layer": 6, "max_source_mass": 0.3442404866218567, "mixer_index": 11, "normalized_entropy": 0.9995797628070386, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3442404866218567, "median": 0.34447744488716125, "p05": 0.3311336636543274, "p95": 0.3566625714302063 }, { "label": "completed_group_1", "mean": 0.33018797636032104, "median": 0.3304833471775055, "p05": 0.3209139108657837, "p95": 0.338434636592865 }, { "label": "current_group_partial", "mean": 0.32557153701782227, "median": 0.32545968890190125, "p05": 0.3193055987358093, "p95": 0.3318970203399658 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0984500646591187, "group": 2, "latest_source_mass": 0.3301700949668884, "layer": 7, "max_source_mass": 0.34007447957992554, "mixer_index": 12, "normalized_entropy": 0.9998523373435156, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.34007447957992554, "median": 0.34058719873428345, "p05": 0.3314547538757324, "p95": 0.34787896275520325 }, { "label": "completed_group_1", "mean": 0.32975539565086365, "median": 0.3295257091522217, "p05": 0.324510782957077, "p95": 0.3352199196815491 }, { "label": "current_group_partial", "mean": 0.3301700949668884, "median": 0.3300544023513794, "p05": 0.3263310492038727, "p95": 0.3340227007865906 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0982636213302612, "group": 2, "latest_source_mass": 0.3256300091743469, "layer": 7, "max_source_mass": 0.3434537351131439, "mixer_index": 13, "normalized_entropy": 0.9996826293120467, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3434537351131439, "median": 0.3430320620536804, "p05": 0.3340873420238495, "p95": 0.35414862632751465 }, { "label": "completed_group_1", "mean": 0.33091625571250916, "median": 0.33138126134872437, "p05": 0.3248554766178131, "p95": 0.335583359003067 }, { "label": "current_group_partial", "mean": 0.3256300091743469, "median": 0.3252079486846924, "p05": 0.3196634352207184, "p95": 0.3317061960697174 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0984963178634644, "group": 2, "latest_source_mass": 0.32797664403915405, "layer": 8, "max_source_mass": 0.33822712302207947, "mixer_index": 14, "normalized_entropy": 0.9998944388244683, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.33822712302207947, "median": 0.3383355736732483, "p05": 0.33158448338508606, "p95": 0.3443746566772461 }, { "label": "completed_group_1", "mean": 0.33379626274108887, "median": 0.3336103558540344, "p05": 0.3301801085472107, "p95": 0.337948203086853 }, { "label": "current_group_partial", "mean": 0.32797664403915405, "median": 0.3281283378601074, "p05": 0.3242723345756531, "p95": 0.3310714364051819 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0980933904647827, "group": 2, "latest_source_mass": 0.3236028254032135, "layer": 8, "max_source_mass": 0.34460341930389404, "mixer_index": 15, "normalized_entropy": 0.9995276785007056, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.34460341930389404, "median": 0.34293144941329956, "p05": 0.33235934376716614, "p95": 0.35933688282966614 }, { "label": "completed_group_1", "mean": 0.33179372549057007, "median": 0.3316723704338074, "p05": 0.32370349764823914, "p95": 0.33980825543403625 }, { "label": "current_group_partial", "mean": 0.3236028254032135, "median": 0.3241952061653137, "p05": 0.3152182102203369, "p95": 0.3305118680000305 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0985530614852905, "group": 3, "latest_source_mass": 0.32961612939834595, "layer": 9, "max_source_mass": 0.3358767628669739, "mixer_index": 16, "normalized_entropy": 0.9999460890949153, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3358767628669739, "median": 0.335875540971756, "p05": 0.3299671709537506, "p95": 0.3413305878639221 }, { "label": "completed_group_1", "mean": 0.3345070481300354, "median": 0.3343847990036011, "p05": 0.33154726028442383, "p95": 0.337911993265152 }, { "label": "completed_group_2", "mean": 0.32961612939834595, "median": 0.3299309015274048, "p05": 0.3261342942714691, "p95": 0.3323304057121277 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3860312700271606, "group": 3, "latest_source_mass": 0.24687285721302032, "layer": 9, "max_source_mass": 0.25749218463897705, "mixer_index": 17, "normalized_entropy": 0.9998102198926082, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25749218463897705, "median": 0.2577459216117859, "p05": 0.25051605701446533, "p95": 0.26390716433525085 }, { "label": "completed_group_1", "mean": 0.24992282688617706, "median": 0.2488851249217987, "p05": 0.24565663933753967, "p95": 0.25723880529403687 }, { "label": "completed_group_2", "mean": 0.24571210145950317, "median": 0.24514061212539673, "p05": 0.24144421517848969, "p95": 0.251026451587677 }, { "label": "current_group_partial", "mean": 0.24687285721302032, "median": 0.24738571047782898, "p05": 0.24093922972679138, "p95": 0.25144800543785095 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862019777297974, "group": 3, "latest_source_mass": 0.2477031946182251, "layer": 10, "max_source_mass": 0.25314557552337646, "mixer_index": 18, "normalized_entropy": 0.999933359470626, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25314557552337646, "median": 0.2529510259628296, "p05": 0.2480011284351349, "p95": 0.25820392370224 }, { "label": "completed_group_1", "mean": 0.25236955285072327, "median": 0.25230497121810913, "p05": 0.2500428557395935, "p95": 0.2550833523273468 }, { "label": "completed_group_2", "mean": 0.24678170680999756, "median": 0.24716459214687347, "p05": 0.24367791414260864, "p95": 0.2489454448223114 }, { "label": "current_group_partial", "mean": 0.2477031946182251, "median": 0.247479647397995, "p05": 0.24608317017555237, "p95": 0.24990878999233246 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.385892391204834, "group": 3, "latest_source_mass": 0.24460072815418243, "layer": 10, "max_source_mass": 0.25815320014953613, "mixer_index": 19, "normalized_entropy": 0.9997100399984806, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25815320014953613, "median": 0.2583469748497009, "p05": 0.24714797735214233, "p95": 0.26989418268203735 }, { "label": "completed_group_1", "mean": 0.25234687328338623, "median": 0.25168517231941223, "p05": 0.24759340286254883, "p95": 0.25932571291923523 }, { "label": "completed_group_2", "mean": 0.2448991984128952, "median": 0.24570998549461365, "p05": 0.23882809281349182, "p95": 0.24954037368297577 }, { "label": "current_group_partial", "mean": 0.24460072815418243, "median": 0.24506908655166626, "p05": 0.23951184749603271, "p95": 0.2482105791568756 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3861613273620605, "group": 3, "latest_source_mass": 0.24515381455421448, "layer": 11, "max_source_mass": 0.25378289818763733, "mixer_index": 20, "normalized_entropy": 0.9999040364286539, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25378289818763733, "median": 0.25346964597702026, "p05": 0.2502344846725464, "p95": 0.25824105739593506 }, { "label": "completed_group_1", "mean": 0.25339174270629883, "median": 0.2532607913017273, "p05": 0.2513120174407959, "p95": 0.25618526339530945 }, { "label": "completed_group_2", "mean": 0.24767157435417175, "median": 0.24778661131858826, "p05": 0.24526026844978333, "p95": 0.24950486421585083 }, { "label": "current_group_partial", "mean": 0.24515381455421448, "median": 0.2451559603214264, "p05": 0.24296040832996368, "p95": 0.2471310794353485 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3855845928192139, "group": 3, "latest_source_mass": 0.24013002216815948, "layer": 11, "max_source_mass": 0.2592235505580902, "mixer_index": 21, "normalized_entropy": 0.9994880103962168, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2592235505580902, "median": 0.260243684053421, "p05": 0.2400972694158554, "p95": 0.27410486340522766 }, { "label": "completed_group_1", "mean": 0.2543467879295349, "median": 0.2532445192337036, "p05": 0.2488831728696823, "p95": 0.26358044147491455 }, { "label": "completed_group_2", "mean": 0.2462996244430542, "median": 0.24627387523651123, "p05": 0.23948119580745697, "p95": 0.2534843385219574 }, { "label": "current_group_partial", "mean": 0.24013002216815948, "median": 0.24106061458587646, "p05": 0.2340238392353058, "p95": 0.24541442096233368 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3861030340194702, "group": 3, "latest_source_mass": 0.24375030398368835, "layer": 12, "max_source_mass": 0.25586679577827454, "mixer_index": 22, "normalized_entropy": 0.999861986670518, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25138330459594727, "median": 0.2519329786300659, "p05": 0.24518054723739624, "p95": 0.25645288825035095 }, { "label": "completed_group_1", "mean": 0.25586679577827454, "median": 0.2557966709136963, "p05": 0.25358980894088745, "p95": 0.25876402854919434 }, { "label": "completed_group_2", "mean": 0.24899962544441223, "median": 0.24906687438488007, "p05": 0.2462620884180069, "p95": 0.2515370845794678 }, { "label": "current_group_partial", "mean": 0.24375030398368835, "median": 0.24418899416923523, "p05": 0.2408682256937027, "p95": 0.24600279331207275 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3856689929962158, "group": 3, "latest_source_mass": 0.24451105296611786, "layer": 12, "max_source_mass": 0.2574368715286255, "mixer_index": 23, "normalized_entropy": 0.9995488922546222, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2574368715286255, "median": 0.25917017459869385, "p05": 0.23716294765472412, "p95": 0.27229955792427063 }, { "label": "completed_group_1", "mean": 0.25204822421073914, "median": 0.2513560652732849, "p05": 0.24607181549072266, "p95": 0.2608553171157837 }, { "label": "completed_group_2", "mean": 0.2460038810968399, "median": 0.2455829679965973, "p05": 0.2395436018705368, "p95": 0.2544419467449188 }, { "label": "current_group_partial", "mean": 0.24451105296611786, "median": 0.2424975037574768, "p05": 0.2388056218624115, "p95": 0.2537745237350464 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.386252999305725, "group": 4, "latest_source_mass": 0.24712973833084106, "layer": 13, "max_source_mass": 0.25257009267807007, "mixer_index": 24, "normalized_entropy": 0.9999701637579107, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25002047419548035, "median": 0.24998526275157928, "p05": 0.24717272818088531, "p95": 0.2532656788825989 }, { "label": "completed_group_1", "mean": 0.25257009267807007, "median": 0.2525791525840759, "p05": 0.25114181637763977, "p95": 0.2538524568080902 }, { "label": "completed_group_2", "mean": 0.25027966499328613, "median": 0.25028353929519653, "p05": 0.24859410524368286, "p95": 0.2516433000564575 }, { "label": "completed_group_3", "mean": 0.24712973833084106, "median": 0.24733176827430725, "p05": 0.2456049621105194, "p95": 0.24814137816429138 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.608945608139038, "group": 4, "latest_source_mass": 0.19259600341320038, "layer": 13, "max_source_mass": 0.2066430151462555, "mixer_index": 25, "normalized_entropy": 0.9996941141430441, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20495271682739258, "median": 0.20551061630249023, "p05": 0.19701695442199707, "p95": 0.21219341456890106 }, { "label": "completed_group_1", "mean": 0.2066430151462555, "median": 0.2066696584224701, "p05": 0.20182059705257416, "p95": 0.21241214871406555 }, { "label": "completed_group_2", "mean": 0.2000001221895218, "median": 0.1994028240442276, "p05": 0.1966598778963089, "p95": 0.20397058129310608 }, { "label": "completed_group_3", "mean": 0.19580815732479095, "median": 0.19627662003040314, "p05": 0.1906815767288208, "p95": 0.20030993223190308 }, { "label": "current_group_partial", "mean": 0.19259600341320038, "median": 0.1924200803041458, "p05": 0.18821945786476135, "p95": 0.19660082459449768 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093038320541382, "group": 4, "latest_source_mass": 0.19651609659194946, "layer": 14, "max_source_mass": 0.20420849323272705, "mixer_index": 26, "normalized_entropy": 0.9999166911758905, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20154938101768494, "median": 0.2015763819217682, "p05": 0.19716835021972656, "p95": 0.20529012382030487 }, { "label": "completed_group_1", "mean": 0.20420849323272705, "median": 0.20418605208396912, "p05": 0.20256520807743073, "p95": 0.20618648827075958 }, { "label": "completed_group_2", "mean": 0.2010107934474945, "median": 0.20109188556671143, "p05": 0.19940726459026337, "p95": 0.2023579627275467 }, { "label": "completed_group_3", "mean": 0.19671522080898285, "median": 0.19701939821243286, "p05": 0.19455352425575256, "p95": 0.19804880023002625 }, { "label": "current_group_partial", "mean": 0.19651609659194946, "median": 0.1964191049337387, "p05": 0.19523656368255615, "p95": 0.19822117686271667 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6085759401321411, "group": 4, "latest_source_mass": 0.18957461416721344, "layer": 14, "max_source_mass": 0.209174245595932, "mixer_index": 27, "normalized_entropy": 0.99946442649617, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20549985766410828, "median": 0.20732679963111877, "p05": 0.1906183660030365, "p95": 0.21469318866729736 }, { "label": "completed_group_1", "mean": 0.209174245595932, "median": 0.2091023325920105, "p05": 0.2042047083377838, "p95": 0.21456730365753174 }, { "label": "completed_group_2", "mean": 0.20157773792743683, "median": 0.20102596282958984, "p05": 0.19761772453784943, "p95": 0.2060481607913971 }, { "label": "completed_group_3", "mean": 0.19417354464530945, "median": 0.19437026977539062, "p05": 0.18892596662044525, "p95": 0.1989399641752243 }, { "label": "current_group_partial", "mean": 0.18957461416721344, "median": 0.18890255689620972, "p05": 0.18614861369132996, "p95": 0.19485054910182953 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093413829803467, "group": 4, "latest_source_mass": 0.1961907148361206, "layer": 15, "max_source_mass": 0.20404505729675293, "mixer_index": 28, "normalized_entropy": 0.999940022878169, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20104943215847015, "median": 0.20089785754680634, "p05": 0.19876199960708618, "p95": 0.20387773215770721 }, { "label": "completed_group_1", "mean": 0.20404505729675293, "median": 0.20400074124336243, "p05": 0.20308127999305725, "p95": 0.20513048768043518 }, { "label": "completed_group_2", "mean": 0.2002137303352356, "median": 0.20016872882843018, "p05": 0.1987290233373642, "p95": 0.2015499770641327 }, { "label": "completed_group_3", "mean": 0.19850105047225952, "median": 0.19858190417289734, "p05": 0.19726355373859406, "p95": 0.19943322241306305 }, { "label": "current_group_partial", "mean": 0.1961907148361206, "median": 0.19618821144104004, "p05": 0.1951756328344345, "p95": 0.19696630537509918 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6080443859100342, "group": 4, "latest_source_mass": 0.18532700836658478, "layer": 15, "max_source_mass": 0.21283750236034393, "mixer_index": 29, "normalized_entropy": 0.9991341532883623, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20538949966430664, "median": 0.20632551610469818, "p05": 0.19240552186965942, "p95": 0.21609632670879364 }, { "label": "completed_group_1", "mean": 0.21283750236034393, "median": 0.2127622365951538, "p05": 0.20815539360046387, "p95": 0.21858084201812744 }, { "label": "completed_group_2", "mean": 0.20341980457305908, "median": 0.20320799946784973, "p05": 0.19931574165821075, "p95": 0.2073775827884674 }, { "label": "completed_group_3", "mean": 0.19302619993686676, "median": 0.1941618025302887, "p05": 0.18753090500831604, "p95": 0.1975058764219284 }, { "label": "current_group_partial", "mean": 0.18532700836658478, "median": 0.18489490449428558, "p05": 0.1818811148405075, "p95": 0.19045278429985046 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093816757202148, "group": 4, "latest_source_mass": 0.1984887570142746, "layer": 16, "max_source_mass": 0.20253092050552368, "mixer_index": 30, "normalized_entropy": 0.9999650581650581, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2010989934206009, "median": 0.20145544409751892, "p05": 0.1965426653623581, "p95": 0.20464390516281128 }, { "label": "completed_group_1", "mean": 0.20253092050552368, "median": 0.20251227915287018, "p05": 0.2015175074338913, "p95": 0.20370690524578094 }, { "label": "completed_group_2", "mean": 0.19950580596923828, "median": 0.19937199354171753, "p05": 0.19786886870861053, "p95": 0.20136688649654388 }, { "label": "completed_group_3", "mean": 0.19837552309036255, "median": 0.19810429215431213, "p05": 0.19704630970954895, "p95": 0.20048341155052185 }, { "label": "current_group_partial", "mean": 0.1984887570142746, "median": 0.19826334714889526, "p05": 0.19740399718284607, "p95": 0.19980067014694214 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6076874732971191, "group": 4, "latest_source_mass": 0.18312416970729828, "layer": 16, "max_source_mass": 0.21552258729934692, "mixer_index": 31, "normalized_entropy": 0.9989123910133733, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20581099390983582, "median": 0.2059018313884735, "p05": 0.19574317336082458, "p95": 0.21629589796066284 }, { "label": "completed_group_1", "mean": 0.21552258729934692, "median": 0.2158341407775879, "p05": 0.20979133248329163, "p95": 0.2216476947069168 }, { "label": "completed_group_2", "mean": 0.20295079052448273, "median": 0.2030434012413025, "p05": 0.19869419932365417, "p95": 0.20675882697105408 }, { "label": "completed_group_3", "mean": 0.19259145855903625, "median": 0.1934264600276947, "p05": 0.18681757152080536, "p95": 0.19656985998153687 }, { "label": "current_group_partial", "mean": 0.18312416970729828, "median": 0.18330088257789612, "p05": 0.17835915088653564, "p95": 0.18824028968811035 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.609355092048645, "group": 5, "latest_source_mass": 0.19643527269363403, "layer": 17, "max_source_mass": 0.2033483237028122, "mixer_index": 32, "normalized_entropy": 0.999948540801223, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20059311389923096, "median": 0.20052847266197205, "p05": 0.1975974291563034, "p95": 0.20357753336429596 }, { "label": "completed_group_1", "mean": 0.2033483237028122, "median": 0.20335550606250763, "p05": 0.20221826434135437, "p95": 0.20440047979354858 }, { "label": "completed_group_2", "mean": 0.2011984884738922, "median": 0.2013639658689499, "p05": 0.19996295869350433, "p95": 0.20205698907375336 }, { "label": "completed_group_3", "mean": 0.1984248161315918, "median": 0.1985226571559906, "p05": 0.1971665471792221, "p95": 0.1996813714504242 }, { "label": "completed_group_4", "mean": 0.19643527269363403, "median": 0.19629330933094025, "p05": 0.19542984664440155, "p95": 0.19760140776634216 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7901467084884644, "group": 5, "latest_source_mass": 0.15958338975906372, "layer": 17, "max_source_mass": 0.18124164640903473, "mixer_index": 33, "normalized_entropy": 0.9990999010931498, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17125868797302246, "median": 0.1713503748178482, "p05": 0.16346795856952667, "p95": 0.17907589673995972 }, { "label": "completed_group_1", "mean": 0.18124164640903473, "median": 0.18149954080581665, "p05": 0.17634934186935425, "p95": 0.18695694208145142 }, { "label": "completed_group_2", "mean": 0.17160002887248993, "median": 0.17129477858543396, "p05": 0.16841214895248413, "p95": 0.17520035803318024 }, { "label": "completed_group_3", "mean": 0.1620369851589203, "median": 0.1626986563205719, "p05": 0.15807151794433594, "p95": 0.16494593024253845 }, { "label": "completed_group_4", "mean": 0.15427924692630768, "median": 0.15451020002365112, "p05": 0.1496276557445526, "p95": 0.15884152054786682 }, { "label": "current_group_partial", "mean": 0.15958338975906372, "median": 0.16020870208740234, "p05": 0.1548137664794922, "p95": 0.16307921707630157 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7916979789733887, "group": 5, "latest_source_mass": 0.1646067053079605, "layer": 18, "max_source_mass": 0.16927622258663177, "mixer_index": 34, "normalized_entropy": 0.9999656816354414, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16689208149909973, "median": 0.1665830761194229, "p05": 0.16523399949073792, "p95": 0.16929879784584045 }, { "label": "completed_group_1", "mean": 0.16927622258663177, "median": 0.16933584213256836, "p05": 0.16805575788021088, "p95": 0.17042185366153717 }, { "label": "completed_group_2", "mean": 0.1683787703514099, "median": 0.16833901405334473, "p05": 0.16767027974128723, "p95": 0.16930325329303741 }, { "label": "completed_group_3", "mean": 0.16569086909294128, "median": 0.1657993346452713, "p05": 0.16471867263317108, "p95": 0.16646946966648102 }, { "label": "completed_group_4", "mean": 0.1651553511619568, "median": 0.16505351662635803, "p05": 0.16424953937530518, "p95": 0.16621217131614685 }, { "label": "current_group_partial", "mean": 0.1646067053079605, "median": 0.16458091139793396, "p05": 0.16394029557704926, "p95": 0.1652415692806244 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.789665699005127, "group": 5, "latest_source_mass": 0.15269872546195984, "layer": 18, "max_source_mass": 0.18269017338752747, "mixer_index": 35, "normalized_entropy": 0.9988314445890273, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1720278114080429, "median": 0.17178001999855042, "p05": 0.1651601344347, "p95": 0.18030256032943726 }, { "label": "completed_group_1", "mean": 0.18269017338752747, "median": 0.1828061193227768, "p05": 0.1778087317943573, "p95": 0.18853498995304108 }, { "label": "completed_group_2", "mean": 0.17356479167938232, "median": 0.17310965061187744, "p05": 0.1708383560180664, "p95": 0.17728117108345032 }, { "label": "completed_group_3", "mean": 0.16241811215877533, "median": 0.16282695531845093, "p05": 0.15827099978923798, "p95": 0.16520199179649353 }, { "label": "completed_group_4", "mean": 0.15660037100315094, "median": 0.15704499185085297, "p05": 0.15247587859630585, "p95": 0.16102959215641022 }, { "label": "current_group_partial", "mean": 0.15269872546195984, "median": 0.15324696898460388, "p05": 0.14927825331687927, "p95": 0.15543338656425476 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917176485061646, "group": 5, "latest_source_mass": 0.16697046160697937, "layer": 19, "max_source_mass": 0.16854418814182281, "mixer_index": 36, "normalized_entropy": 0.9999766594107029, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.16644492745399475, "median": 0.16678380966186523, "p05": 0.16287505626678467, "p95": 0.1690630316734314 }, { "label": "completed_group_1", "mean": 0.16854418814182281, "median": 0.16858378052711487, "p05": 0.1675834059715271, "p95": 0.1694023609161377 }, { "label": "completed_group_2", "mean": 0.16665583848953247, "median": 0.16633233428001404, "p05": 0.16533653438091278, "p95": 0.16844478249549866 }, { "label": "completed_group_3", "mean": 0.16677430272102356, "median": 0.1665923297405243, "p05": 0.16590546071529388, "p95": 0.16807952523231506 }, { "label": "completed_group_4", "mean": 0.16461026668548584, "median": 0.1646481156349182, "p05": 0.16389033198356628, "p95": 0.1651972234249115 }, { "label": "current_group_partial", "mean": 0.16697046160697937, "median": 0.166935995221138, "p05": 0.16631023585796356, "p95": 0.16765466332435608 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7893871068954468, "group": 5, "latest_source_mass": 0.15126986801624298, "layer": 19, "max_source_mass": 0.18359911441802979, "mixer_index": 37, "normalized_entropy": 0.9986759593721415, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.17303554713726044, "median": 0.17261022329330444, "p05": 0.16660651564598083, "p95": 0.18199719488620758 }, { "label": "completed_group_1", "mean": 0.18359911441802979, "median": 0.18389560282230377, "p05": 0.17790013551712036, "p95": 0.1901787668466568 }, { "label": "completed_group_2", "mean": 0.17276406288146973, "median": 0.17272299528121948, "p05": 0.16975931823253632, "p95": 0.1759810596704483 }, { "label": "completed_group_3", "mean": 0.16385790705680847, "median": 0.16339454054832458, "p05": 0.16003964841365814, "p95": 0.1676909327507019 }, { "label": "completed_group_4", "mean": 0.1554735004901886, "median": 0.15558910369873047, "p05": 0.15120893716812134, "p95": 0.15934500098228455 }, { "label": "current_group_partial", "mean": 0.15126986801624298, "median": 0.15185734629631042, "p05": 0.14759114384651184, "p95": 0.1540675163269043 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917413711547852, "group": 5, "latest_source_mass": 0.16742712259292603, "layer": 20, "max_source_mass": 0.16742712259292603, "mixer_index": 38, "normalized_entropy": 0.999989899272988, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16618916392326355, "median": 0.16628703474998474, "p05": 0.1636679470539093, "p95": 0.16814759373664856 }, { "label": "completed_group_1", "mean": 0.16740113496780396, "median": 0.16743052005767822, "p05": 0.16664837300777435, "p95": 0.16812929511070251 }, { "label": "completed_group_2", "mean": 0.16675519943237305, "median": 0.16677126288414001, "p05": 0.1660051941871643, "p95": 0.16743934154510498 }, { "label": "completed_group_3", "mean": 0.1668717861175537, "median": 0.16686314344406128, "p05": 0.16629263758659363, "p95": 0.1675407886505127 }, { "label": "completed_group_4", "mean": 0.1653555929660797, "median": 0.16525551676750183, "p05": 0.16465890407562256, "p95": 0.166346937417984 }, { "label": "current_group_partial", "mean": 0.16742712259292603, "median": 0.1672172248363495, "p05": 0.16667790710926056, "p95": 0.16856777667999268 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7882882356643677, "group": 5, "latest_source_mass": 0.14739990234375, "layer": 20, "max_source_mass": 0.18687990307807922, "mixer_index": 39, "normalized_entropy": 0.9980626676608648, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.17288538813591003, "median": 0.17220303416252136, "p05": 0.16650697588920593, "p95": 0.18239524960517883 }, { "label": "completed_group_1", "mean": 0.18687990307807922, "median": 0.1869410276412964, "p05": 0.1804148554801941, "p95": 0.19480778276920319 }, { "label": "completed_group_2", "mean": 0.17607632279396057, "median": 0.17579063773155212, "p05": 0.1729249656200409, "p95": 0.18001750111579895 }, { "label": "completed_group_3", "mean": 0.16286307573318481, "median": 0.162565678358078, "p05": 0.15797676146030426, "p95": 0.16752192378044128 }, { "label": "completed_group_4", "mean": 0.15389540791511536, "median": 0.15402349829673767, "p05": 0.14864680171012878, "p95": 0.1582801640033722 }, { "label": "current_group_partial", "mean": 0.14739990234375, "median": 0.14793044328689575, "p05": 0.14291448891162872, "p95": 0.15072257816791534 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917335033416748, "group": 6, "latest_source_mass": 0.16607257723808289, "layer": 21, "max_source_mass": 0.16799162328243256, "mixer_index": 40, "normalized_entropy": 0.9999855081628835, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16660980880260468, "median": 0.16617761552333832, "p05": 0.1644040048122406, "p95": 0.1693679392337799 }, { "label": "completed_group_1", "mean": 0.16799162328243256, "median": 0.1680111587047577, "p05": 0.16704340279102325, "p95": 0.16876636445522308 }, { "label": "completed_group_2", "mean": 0.16742658615112305, "median": 0.16741225123405457, "p05": 0.1667558252811432, "p95": 0.16810807585716248 }, { "label": "completed_group_3", "mean": 0.1664734035730362, "median": 0.1665918529033661, "p05": 0.16557857394218445, "p95": 0.16731621325016022 }, { "label": "completed_group_4", "mean": 0.16542600095272064, "median": 0.16541361808776855, "p05": 0.16449882090091705, "p95": 0.166628897190094 }, { "label": "completed_group_5", "mean": 0.16607257723808289, "median": 0.166090726852417, "p05": 0.16493722796440125, "p95": 0.16713958978652954 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.941649079322815, "group": 6, "latest_source_mass": 0.13102038204669952, "layer": 21, "max_source_mass": 0.16432973742485046, "mixer_index": 41, "normalized_entropy": 0.9978102433277473, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15037035942077637, "median": 0.15097856521606445, "p05": 0.1399948000907898, "p95": 0.1612456738948822 }, { "label": "completed_group_1", "mean": 0.16432973742485046, "median": 0.16473636031150818, "p05": 0.15782266855239868, "p95": 0.17067503929138184 }, { "label": "completed_group_2", "mean": 0.15334899723529816, "median": 0.15358540415763855, "p05": 0.1502012014389038, "p95": 0.1558559238910675 }, { "label": "completed_group_3", "mean": 0.1418340653181076, "median": 0.14079436659812927, "p05": 0.13714224100112915, "p95": 0.14798131585121155 }, { "label": "completed_group_4", "mean": 0.13212506473064423, "median": 0.13160677254199982, "p05": 0.1273195594549179, "p95": 0.13741742074489594 }, { "label": "completed_group_5", "mean": 0.12697139382362366, "median": 0.12719248235225677, "p05": 0.1226799264550209, "p95": 0.1304648220539093 }, { "label": "current_group_partial", "mean": 0.13102038204669952, "median": 0.1311827003955841, "p05": 0.12728165090084076, "p95": 0.134886234998703 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458887577056885, "group": 6, "latest_source_mass": 0.14201194047927856, "layer": 22, "max_source_mass": 0.14392223954200745, "mixer_index": 42, "normalized_entropy": 0.9999890070208868, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1431310474872589, "median": 0.1429322063922882, "p05": 0.14122292399406433, "p95": 0.14551937580108643 }, { "label": "completed_group_1", "mean": 0.14324922859668732, "median": 0.14312732219696045, "p05": 0.14240022003650665, "p95": 0.14425528049468994 }, { "label": "completed_group_2", "mean": 0.14309078454971313, "median": 0.14308927953243256, "p05": 0.14254748821258545, "p95": 0.14363758265972137 }, { "label": "completed_group_3", "mean": 0.142366424202919, "median": 0.1424175649881363, "p05": 0.14179164171218872, "p95": 0.14290453493595123 }, { "label": "completed_group_4", "mean": 0.14222830533981323, "median": 0.14226765930652618, "p05": 0.14123091101646423, "p95": 0.1430286169052124 }, { "label": "completed_group_5", "mean": 0.14392223954200745, "median": 0.14401790499687195, "p05": 0.14277607202529907, "p95": 0.1448441743850708 }, { "label": "current_group_partial", "mean": 0.14201194047927856, "median": 0.1420515477657318, "p05": 0.14140468835830688, "p95": 0.1425478160381317 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9409761428833008, "group": 6, "latest_source_mass": 0.12605468928813934, "layer": 22, "max_source_mass": 0.16567249596118927, "mixer_index": 43, "normalized_entropy": 0.9974644224069606, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1505419909954071, "median": 0.15057668089866638, "p05": 0.1423991620540619, "p95": 0.1602301448583603 }, { "label": "completed_group_1", "mean": 0.16567249596118927, "median": 0.16589392721652985, "p05": 0.15891215205192566, "p95": 0.17315104603767395 }, { "label": "completed_group_2", "mean": 0.1550290882587433, "median": 0.15488041937351227, "p05": 0.15190331637859344, "p95": 0.15830960869789124 }, { "label": "completed_group_3", "mean": 0.14243589341640472, "median": 0.14207899570465088, "p05": 0.13828127086162567, "p95": 0.14758256077766418 }, { "label": "completed_group_4", "mean": 0.13249853253364563, "median": 0.13208040595054626, "p05": 0.12817804515361786, "p95": 0.13691219687461853 }, { "label": "completed_group_5", "mean": 0.12776729464530945, "median": 0.1281416267156601, "p05": 0.1238994225859642, "p95": 0.13073651492595673 }, { "label": "current_group_partial", "mean": 0.12605468928813934, "median": 0.12603645026683807, "p05": 0.12259864807128906, "p95": 0.12950734794139862 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458999633789062, "group": 6, "latest_source_mass": 0.14296774566173553, "layer": 23, "max_source_mass": 0.1435922086238861, "mixer_index": 44, "normalized_entropy": 0.9999947655977786, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14249610900878906, "median": 0.14253944158554077, "p05": 0.14135295152664185, "p95": 0.14354844391345978 }, { "label": "completed_group_1", "mean": 0.14256572723388672, "median": 0.1425473988056183, "p05": 0.14203670620918274, "p95": 0.14315003156661987 }, { "label": "completed_group_2", "mean": 0.1435922086238861, "median": 0.14354602992534637, "p05": 0.142880380153656, "p95": 0.14443112909793854 }, { "label": "completed_group_3", "mean": 0.14210084080696106, "median": 0.14210137724876404, "p05": 0.14155790209770203, "p95": 0.14259196817874908 }, { "label": "completed_group_4", "mean": 0.14355877041816711, "median": 0.14355827867984772, "p05": 0.1432723104953766, "p95": 0.14383304119110107 }, { "label": "completed_group_5", "mean": 0.1427185833454132, "median": 0.1427343189716339, "p05": 0.14239519834518433, "p95": 0.14299128949642181 }, { "label": "current_group_partial", "mean": 0.14296774566173553, "median": 0.14299029111862183, "p05": 0.14251580834388733, "p95": 0.14344699680805206 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.941575527191162, "group": 6, "latest_source_mass": 0.126667320728302, "layer": 23, "max_source_mass": 0.1640411913394928, "mixer_index": 45, "normalized_entropy": 0.997772445009213, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15046924352645874, "median": 0.15056449174880981, "p05": 0.1412365734577179, "p95": 0.16024990379810333 }, { "label": "completed_group_1", "mean": 0.1640411913394928, "median": 0.16397136449813843, "p05": 0.15763643383979797, "p95": 0.17164170742034912 }, { "label": "completed_group_2", "mean": 0.15395452082157135, "median": 0.1541147232055664, "p05": 0.1505676656961441, "p95": 0.157239630818367 }, { "label": "completed_group_3", "mean": 0.14198558032512665, "median": 0.14145654439926147, "p05": 0.13773861527442932, "p95": 0.14743509888648987 }, { "label": "completed_group_4", "mean": 0.13400861620903015, "median": 0.13384035229682922, "p05": 0.1295107901096344, "p95": 0.13838393986225128 }, { "label": "completed_group_5", "mean": 0.1288735270500183, "median": 0.12916186451911926, "p05": 0.12452569603919983, "p95": 0.1320379376411438 }, { "label": "current_group_partial", "mean": 0.126667320728302, "median": 0.12697717547416687, "p05": 0.12317797541618347, "p95": 0.1303042471408844 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458911418914795, "group": 6, "latest_source_mass": 0.14433911442756653, "layer": 24, "max_source_mass": 0.14433911442756653, "mixer_index": 46, "normalized_entropy": 0.9999902322500127, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14260169863700867, "median": 0.1426181197166443, "p05": 0.14112016558647156, "p95": 0.14391711354255676 }, { "label": "completed_group_1", "mean": 0.14355596899986267, "median": 0.14354628324508667, "p05": 0.1428137570619583, "p95": 0.1442246437072754 }, { "label": "completed_group_2", "mean": 0.1428615301847458, "median": 0.14287462830543518, "p05": 0.14239956438541412, "p95": 0.1433551013469696 }, { "label": "completed_group_3", "mean": 0.14244115352630615, "median": 0.14245545864105225, "p05": 0.14187632501125336, "p95": 0.1429358869791031 }, { "label": "completed_group_4", "mean": 0.14214617013931274, "median": 0.14210420846939087, "p05": 0.1417112946510315, "p95": 0.1426859349012375 }, { "label": "completed_group_5", "mean": 0.14205437898635864, "median": 0.14205071330070496, "p05": 0.14159706234931946, "p95": 0.14274871349334717 }, { "label": "current_group_partial", "mean": 0.14433911442756653, "median": 0.14430615305900574, "p05": 0.14394080638885498, "p95": 0.14513593912124634 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9400205612182617, "group": 6, "latest_source_mass": 0.12250205874443054, "layer": 24, "max_source_mass": 0.16689133644104004, "mixer_index": 47, "normalized_entropy": 0.9969733505732982, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1507154405117035, "median": 0.15137067437171936, "p05": 0.13938520848751068, "p95": 0.16102933883666992 }, { "label": "completed_group_1", "mean": 0.16689133644104004, "median": 0.16768822073936462, "p05": 0.15714310109615326, "p95": 0.17547596991062164 }, { "label": "completed_group_2", "mean": 0.15648441016674042, "median": 0.1563897281885147, "p05": 0.15373846888542175, "p95": 0.15962360799312592 }, { "label": "completed_group_3", "mean": 0.1427520215511322, "median": 0.14218726754188538, "p05": 0.13820181787014008, "p95": 0.14905300736427307 }, { "label": "completed_group_4", "mean": 0.13353285193443298, "median": 0.1328810751438141, "p05": 0.1281396746635437, "p95": 0.1398344486951828 }, { "label": "completed_group_5", "mean": 0.12712189555168152, "median": 0.12679056823253632, "p05": 0.12242501974105835, "p95": 0.1326143443584442 }, { "label": "current_group_partial", "mean": 0.12250205874443054, "median": 0.12238125503063202, "p05": 0.11881453543901443, "p95": 0.12615910172462463 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458441734313965, "group": 7, "latest_source_mass": 0.14612418413162231, "layer": 25, "max_source_mass": 0.14612418413162231, "mixer_index": 48, "normalized_entropy": 0.9999660952362324, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1422245353460312, "median": 0.14217790961265564, "p05": 0.14090952277183533, "p95": 0.14379017055034637 }, { "label": "completed_group_1", "mean": 0.14142528176307678, "median": 0.14123442769050598, "p05": 0.1402144730091095, "p95": 0.14290855824947357 }, { "label": "completed_group_2", "mean": 0.14198407530784607, "median": 0.14185458421707153, "p05": 0.14110887050628662, "p95": 0.14297308027744293 }, { "label": "completed_group_3", "mean": 0.14165858924388885, "median": 0.14167557656764984, "p05": 0.1411055624485016, "p95": 0.14217206835746765 }, { "label": "completed_group_4", "mean": 0.14288201928138733, "median": 0.14292356371879578, "p05": 0.1422932744026184, "p95": 0.14346298575401306 }, { "label": "completed_group_5", "mean": 0.14370128512382507, "median": 0.14391976594924927, "p05": 0.14265118539333344, "p95": 0.14442476630210876 }, { "label": "completed_group_6", "mean": 0.14612418413162231, "median": 0.14619040489196777, "p05": 0.14509475231170654, "p95": 0.14701899886131287 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.0741422176361084, "group": 7, "latest_source_mass": 0.11380299180746078, "layer": 25, "max_source_mass": 0.14654947817325592, "mixer_index": 49, "normalized_entropy": 0.9974515638273503, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13300859928131104, "median": 0.13330551981925964, "p05": 0.12400922924280167, "p95": 0.14149883389472961 }, { "label": "completed_group_1", "mean": 0.14654947817325592, "median": 0.14719218015670776, "p05": 0.1369621753692627, "p95": 0.15508656203746796 }, { "label": "completed_group_2", "mean": 0.1379268318414688, "median": 0.1377311646938324, "p05": 0.13488081097602844, "p95": 0.14173594117164612 }, { "label": "completed_group_3", "mean": 0.12711459398269653, "median": 0.12693974375724792, "p05": 0.12290681898593903, "p95": 0.131547749042511 }, { "label": "completed_group_4", "mean": 0.1185426190495491, "median": 0.118228018283844, "p05": 0.11395476758480072, "p95": 0.12371979653835297 }, { "label": "completed_group_5", "mean": 0.1134917140007019, "median": 0.11304019391536713, "p05": 0.10924124717712402, "p95": 0.1182163655757904 }, { "label": "completed_group_6", "mean": 0.10956315696239471, "median": 0.10938596725463867, "p05": 0.10588769614696503, "p95": 0.11350928992033005 }, { "label": "current_group_partial", "mean": 0.11380299180746078, "median": 0.11401359736919403, "p05": 0.1090584248304367, "p95": 0.11826062947511673 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0794055461883545, "group": 7, "latest_source_mass": 0.1260453760623932, "layer": 26, "max_source_mass": 0.1263071894645691, "mixer_index": 50, "normalized_entropy": 0.9999826898276486, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12428364902734756, "median": 0.1242905929684639, "p05": 0.12320776283740997, "p95": 0.12528444826602936 }, { "label": "completed_group_1", "mean": 0.123402900993824, "median": 0.12333650141954422, "p05": 0.12251013517379761, "p95": 0.12441572546958923 }, { "label": "completed_group_2", "mean": 0.12436956912279129, "median": 0.1244126409292221, "p05": 0.12380266189575195, "p95": 0.12485788762569427 }, { "label": "completed_group_3", "mean": 0.12428067624568939, "median": 0.12426625192165375, "p05": 0.12392986565828323, "p95": 0.1246793270111084 }, { "label": "completed_group_4", "mean": 0.12555459141731262, "median": 0.12557968497276306, "p05": 0.1250462681055069, "p95": 0.12604334950447083 }, { "label": "completed_group_5", "mean": 0.12575605511665344, "median": 0.1258038580417633, "p05": 0.12512780725955963, "p95": 0.12622345983982086 }, { "label": "completed_group_6", "mean": 0.1263071894645691, "median": 0.12632015347480774, "p05": 0.12580350041389465, "p95": 0.12684133648872375 }, { "label": "current_group_partial", "mean": 0.1260453760623932, "median": 0.12603351473808289, "p05": 0.12576976418495178, "p95": 0.12635180354118347 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.071772813796997, "group": 7, "latest_source_mass": 0.1069779098033905, "layer": 26, "max_source_mass": 0.1495765745639801, "mixer_index": 51, "normalized_entropy": 0.9963121214378339, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.134201318025589, "median": 0.13547956943511963, "p05": 0.12102854251861572, "p95": 0.1446569263935089 }, { "label": "completed_group_1", "mean": 0.1495765745639801, "median": 0.15066714584827423, "p05": 0.13745656609535217, "p95": 0.15931519865989685 }, { "label": "completed_group_2", "mean": 0.14145472645759583, "median": 0.14118221402168274, "p05": 0.13830184936523438, "p95": 0.14542292058467865 }, { "label": "completed_group_3", "mean": 0.12840478122234344, "median": 0.12756571173667908, "p05": 0.1237710565328598, "p95": 0.1351458877325058 }, { "label": "completed_group_4", "mean": 0.11872729659080505, "median": 0.1178978681564331, "p05": 0.11296786367893219, "p95": 0.1259407103061676 }, { "label": "completed_group_5", "mean": 0.11272042989730835, "median": 0.11211050301790237, "p05": 0.10776206851005554, "p95": 0.11910594999790192 }, { "label": "completed_group_6", "mean": 0.10793697834014893, "median": 0.10766834765672684, "p05": 0.10393505543470383, "p95": 0.11274770647287369 }, { "label": "current_group_partial", "mean": 0.1069779098033905, "median": 0.10693258047103882, "p05": 0.10282129794359207, "p95": 0.11074373126029968 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079404592514038, "group": 7, "latest_source_mass": 0.12623761594295502, "layer": 27, "max_source_mass": 0.1266230344772339, "mixer_index": 52, "normalized_entropy": 0.9999822312072463, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12455718219280243, "median": 0.12448663264513016, "p05": 0.12370531260967255, "p95": 0.12576276063919067 }, { "label": "completed_group_1", "mean": 0.12374450266361237, "median": 0.12364563345909119, "p05": 0.12273170799016953, "p95": 0.12496861815452576 }, { "label": "completed_group_2", "mean": 0.12391386926174164, "median": 0.12387685477733612, "p05": 0.12311351299285889, "p95": 0.12478289753198624 }, { "label": "completed_group_3", "mean": 0.12458804249763489, "median": 0.12460290640592575, "p05": 0.12419690191745758, "p95": 0.12499304115772247 }, { "label": "completed_group_4", "mean": 0.1251734495162964, "median": 0.12516582012176514, "p05": 0.12470955401659012, "p95": 0.12564992904663086 }, { "label": "completed_group_5", "mean": 0.1251622885465622, "median": 0.12524060904979706, "p05": 0.12443183362483978, "p95": 0.12575514614582062 }, { "label": "completed_group_6", "mean": 0.1266230344772339, "median": 0.12661367654800415, "p05": 0.12584306299686432, "p95": 0.12732140719890594 }, { "label": "current_group_partial", "mean": 0.12623761594295502, "median": 0.12622185051441193, "p05": 0.12569603323936462, "p95": 0.1268007904291153 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0732176303863525, "group": 7, "latest_source_mass": 0.1073671281337738, "layer": 27, "max_source_mass": 0.1473538726568222, "mixer_index": 53, "normalized_entropy": 0.9970069313473197, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13281872868537903, "median": 0.13317491114139557, "p05": 0.12417874485254288, "p95": 0.14101722836494446 }, { "label": "completed_group_1", "mean": 0.1473538726568222, "median": 0.14751407504081726, "p05": 0.13885104656219482, "p95": 0.15571141242980957 }, { "label": "completed_group_2", "mean": 0.13968363404273987, "median": 0.13911551237106323, "p05": 0.1365838497877121, "p95": 0.14400556683540344 }, { "label": "completed_group_3", "mean": 0.12848341464996338, "median": 0.12823450565338135, "p05": 0.12424321472644806, "p95": 0.13310366868972778 }, { "label": "completed_group_4", "mean": 0.12016645073890686, "median": 0.11972066760063171, "p05": 0.11583883315324783, "p95": 0.12517021596431732 }, { "label": "completed_group_5", "mean": 0.11505335569381714, "median": 0.11486441642045975, "p05": 0.11096000671386719, "p95": 0.11946934461593628 }, { "label": "completed_group_6", "mean": 0.10907343029975891, "median": 0.1089182049036026, "p05": 0.10512679070234299, "p95": 0.11236356198787689 }, { "label": "current_group_partial", "mean": 0.1073671281337738, "median": 0.10785426199436188, "p05": 0.10343525558710098, "p95": 0.11085459589958191 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0794081687927246, "group": 7, "latest_source_mass": 0.12505511939525604, "layer": 28, "max_source_mass": 0.12684063613414764, "mixer_index": 54, "normalized_entropy": 0.9999839510337549, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12450858950614929, "median": 0.12455325573682785, "p05": 0.12283766269683838, "p95": 0.12648260593414307 }, { "label": "completed_group_1", "mean": 0.12424901872873306, "median": 0.12403430044651031, "p05": 0.12319816648960114, "p95": 0.1257738173007965 }, { "label": "completed_group_2", "mean": 0.12409762293100357, "median": 0.12406422197818756, "p05": 0.12322939932346344, "p95": 0.12511804699897766 }, { "label": "completed_group_3", "mean": 0.12468484044075012, "median": 0.12464898824691772, "p05": 0.12426739186048508, "p95": 0.12527169287204742 }, { "label": "completed_group_4", "mean": 0.12504075467586517, "median": 0.1250670850276947, "p05": 0.1244644820690155, "p95": 0.12563633918762207 }, { "label": "completed_group_5", "mean": 0.1255234032869339, "median": 0.12571458518505096, "p05": 0.12471026182174683, "p95": 0.12614558637142181 }, { "label": "completed_group_6", "mean": 0.12684063613414764, "median": 0.12695147097110748, "p05": 0.12606506049633026, "p95": 0.1274568736553192 }, { "label": "current_group_partial", "mean": 0.12505511939525604, "median": 0.12517717480659485, "p05": 0.12436046451330185, "p95": 0.1254468858242035 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.072478771209717, "group": 7, "latest_source_mass": 0.10563355684280396, "layer": 28, "max_source_mass": 0.14877822995185852, "mixer_index": 55, "normalized_entropy": 0.996651615190637, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13302236795425415, "median": 0.13431280851364136, "p05": 0.11974859982728958, "p95": 0.14355170726776123 }, { "label": "completed_group_1", "mean": 0.14877822995185852, "median": 0.1492476761341095, "p05": 0.13942936062812805, "p95": 0.15665946900844574 }, { "label": "completed_group_2", "mean": 0.14031866192817688, "median": 0.14013901352882385, "p05": 0.13727493584156036, "p95": 0.14390599727630615 }, { "label": "completed_group_3", "mean": 0.12882456183433533, "median": 0.12811559438705444, "p05": 0.12456346303224564, "p95": 0.13553638756275177 }, { "label": "completed_group_4", "mean": 0.11896270513534546, "median": 0.1182594895362854, "p05": 0.11376718431711197, "p95": 0.12510910630226135 }, { "label": "completed_group_5", "mean": 0.1144586130976677, "median": 0.11415413022041321, "p05": 0.10992536693811417, "p95": 0.11974817514419556 }, { "label": "completed_group_6", "mean": 0.11000129580497742, "median": 0.10966727137565613, "p05": 0.10602983832359314, "p95": 0.11363138258457184 }, { "label": "current_group_partial", "mean": 0.10563355684280396, "median": 0.10599470138549805, "p05": 0.10157415270805359, "p95": 0.10950814187526703 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079291820526123, "group": 8, "latest_source_mass": 0.12825238704681396, "layer": 29, "max_source_mass": 0.12825238704681396, "mixer_index": 56, "normalized_entropy": 0.9999279993446741, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12426379323005676, "median": 0.12415257841348648, "p05": 0.12234964966773987, "p95": 0.12625235319137573 }, { "label": "completed_group_1", "mean": 0.12249694764614105, "median": 0.12249137461185455, "p05": 0.12120498716831207, "p95": 0.12380178272724152 }, { "label": "completed_group_2", "mean": 0.12262053787708282, "median": 0.12276404350996017, "p05": 0.1215757429599762, "p95": 0.12327249348163605 }, { "label": "completed_group_3", "mean": 0.12315946817398071, "median": 0.12308797240257263, "p05": 0.12251235544681549, "p95": 0.12412448227405548 }, { "label": "completed_group_4", "mean": 0.1255876123905182, "median": 0.1256473809480667, "p05": 0.12454833835363388, "p95": 0.12639300525188446 }, { "label": "completed_group_5", "mean": 0.1262608915567398, "median": 0.12635794281959534, "p05": 0.12536443769931793, "p95": 0.12692558765411377 }, { "label": "completed_group_6", "mean": 0.1273583620786667, "median": 0.12732446193695068, "p05": 0.12652143836021423, "p95": 0.1282241940498352 }, { "label": "completed_group_7", "mean": 0.12825238704681396, "median": 0.12817136943340302, "p05": 0.1276511698961258, "p95": 0.1291390061378479 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1905503273010254, "group": 8, "latest_source_mass": 0.10074523091316223, "layer": 29, "max_source_mass": 0.13242119550704956, "mixer_index": 57, "normalized_entropy": 0.9969624179048253, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11892962455749512, "median": 0.12071715295314789, "p05": 0.10431190580129623, "p95": 0.12862780690193176 }, { "label": "completed_group_1", "mean": 0.13242119550704956, "median": 0.13316701352596283, "p05": 0.12286566942930222, "p95": 0.14010177552700043 }, { "label": "completed_group_2", "mean": 0.12570279836654663, "median": 0.12560519576072693, "p05": 0.12152561545372009, "p95": 0.1298297792673111 }, { "label": "completed_group_3", "mean": 0.11684690415859222, "median": 0.11606287956237793, "p05": 0.11148497462272644, "p95": 0.12368319928646088 }, { "label": "completed_group_4", "mean": 0.10851434618234634, "median": 0.10789132863283157, "p05": 0.10347767174243927, "p95": 0.11500974744558334 }, { "label": "completed_group_5", "mean": 0.10355254262685776, "median": 0.10266663134098053, "p05": 0.09956449270248413, "p95": 0.10929413884878159 }, { "label": "completed_group_6", "mean": 0.09824738651514053, "median": 0.09788262844085693, "p05": 0.09488938748836517, "p95": 0.10226631164550781 }, { "label": "completed_group_7", "mean": 0.0950399786233902, "median": 0.0952744334936142, "p05": 0.09167876094579697, "p95": 0.09785640239715576 }, { "label": "current_group_partial", "mean": 0.10074523091316223, "median": 0.10040169954299927, "p05": 0.09587707370519638, "p95": 0.10640941560268402 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1971964836120605, "group": 8, "latest_source_mass": 0.11228709667921066, "layer": 30, "max_source_mass": 0.11228709667921066, "mixer_index": 58, "normalized_entropy": 0.9999872139951242, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11044953763484955, "median": 0.11043476313352585, "p05": 0.10951471328735352, "p95": 0.11141420900821686 }, { "label": "completed_group_1", "mean": 0.11010265350341797, "median": 0.11030832678079605, "p05": 0.10908854007720947, "p95": 0.11079797893762589 }, { "label": "completed_group_2", "mean": 0.1108412891626358, "median": 0.11078473180532455, "p05": 0.10996625572443008, "p95": 0.11167524755001068 }, { "label": "completed_group_3", "mean": 0.11044170707464218, "median": 0.1103954166173935, "p05": 0.1099170595407486, "p95": 0.11091860383749008 }, { "label": "completed_group_4", "mean": 0.11131919920444489, "median": 0.11133146286010742, "p05": 0.11095823347568512, "p95": 0.11168018728494644 }, { "label": "completed_group_5", "mean": 0.11133328080177307, "median": 0.11128850281238556, "p05": 0.11106889694929123, "p95": 0.11170809715986252 }, { "label": "completed_group_6", "mean": 0.11097346991300583, "median": 0.11084912717342377, "p05": 0.1105891764163971, "p95": 0.11148890852928162 }, { "label": "completed_group_7", "mean": 0.11225177347660065, "median": 0.11221592128276825, "p05": 0.11170009523630142, "p95": 0.11293211579322815 }, { "label": "current_group_partial", "mean": 0.11228709667921066, "median": 0.11222326010465622, "p05": 0.1117643192410469, "p95": 0.11299698054790497 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1902995109558105, "group": 8, "latest_source_mass": 0.09534209221601486, "layer": 30, "max_source_mass": 0.13237422704696655, "mixer_index": 59, "normalized_entropy": 0.9968482664667785, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11926669627428055, "median": 0.12010449916124344, "p05": 0.10881246626377106, "p95": 0.12763431668281555 }, { "label": "completed_group_1", "mean": 0.13237422704696655, "median": 0.1329837143421173, "p05": 0.12264501303434372, "p95": 0.14073863625526428 }, { "label": "completed_group_2", "mean": 0.12644407153129578, "median": 0.1259550154209137, "p05": 0.12286148220300674, "p95": 0.13101284205913544 }, { "label": "completed_group_3", "mean": 0.11611802130937576, "median": 0.11590427160263062, "p05": 0.11185136437416077, "p95": 0.1205722838640213 }, { "label": "completed_group_4", "mean": 0.11051423847675323, "median": 0.11005840450525284, "p05": 0.10631610453128815, "p95": 0.11548890173435211 }, { "label": "completed_group_5", "mean": 0.10466913878917694, "median": 0.10406064987182617, "p05": 0.10079286992549896, "p95": 0.10976865142583847 }, { "label": "completed_group_6", "mean": 0.09992007911205292, "median": 0.09962707757949829, "p05": 0.0964651107788086, "p95": 0.10393689572811127 }, { "label": "completed_group_7", "mean": 0.09535142779350281, "median": 0.09555896371603012, "p05": 0.09170880168676376, "p95": 0.0982663482427597 }, { "label": "current_group_partial", "mean": 0.09534209221601486, "median": 0.0955871120095253, "p05": 0.09114285558462143, "p95": 0.09938932955265045 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.197113037109375, "group": 8, "latest_source_mass": 0.11361998319625854, "layer": 31, "max_source_mass": 0.11361998319625854, "mixer_index": 60, "normalized_entropy": 0.9999492358550897, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11037027835845947, "median": 0.11050257831811905, "p05": 0.10850981622934341, "p95": 0.1116580218076706 }, { "label": "completed_group_1", "mean": 0.10917964577674866, "median": 0.1092953234910965, "p05": 0.10749757289886475, "p95": 0.11104606091976166 }, { "label": "completed_group_2", "mean": 0.10961419343948364, "median": 0.10983817279338837, "p05": 0.10842632502317429, "p95": 0.11050552129745483 }, { "label": "completed_group_3", "mean": 0.10947251319885254, "median": 0.10951714217662811, "p05": 0.10870970040559769, "p95": 0.1103382408618927 }, { "label": "completed_group_4", "mean": 0.11109665781259537, "median": 0.11102215945720673, "p05": 0.11052000522613525, "p95": 0.11176691204309464 }, { "label": "completed_group_5", "mean": 0.11174711585044861, "median": 0.11183997988700867, "p05": 0.11095673590898514, "p95": 0.11240804195404053 }, { "label": "completed_group_6", "mean": 0.1118166595697403, "median": 0.11173175275325775, "p05": 0.11096500605344772, "p95": 0.11265994608402252 }, { "label": "completed_group_7", "mean": 0.11308293789625168, "median": 0.11292627453804016, "p05": 0.11227726191282272, "p95": 0.11403826624155045 }, { "label": "current_group_partial", "mean": 0.11361998319625854, "median": 0.11346748471260071, "p05": 0.11268991231918335, "p95": 0.11467830836772919 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1894307136535645, "group": 8, "latest_source_mass": 0.09481187164783478, "layer": 31, "max_source_mass": 0.13425534963607788, "mixer_index": 61, "normalized_entropy": 0.9964528597745325, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1198490560054779, "median": 0.12130686640739441, "p05": 0.10625117272138596, "p95": 0.12946197390556335 }, { "label": "completed_group_1", "mean": 0.13425534963607788, "median": 0.13473033905029297, "p05": 0.12351738661527634, "p95": 0.14344483613967896 }, { "label": "completed_group_2", "mean": 0.12682178616523743, "median": 0.1263064742088318, "p05": 0.12210550904273987, "p95": 0.13173425197601318 }, { "label": "completed_group_3", "mean": 0.11624304950237274, "median": 0.1158868670463562, "p05": 0.11147508770227432, "p95": 0.12220768630504608 }, { "label": "completed_group_4", "mean": 0.10967147350311279, "median": 0.10917282849550247, "p05": 0.104550302028656, "p95": 0.11562594771385193 }, { "label": "completed_group_5", "mean": 0.10377678275108337, "median": 0.10315801948308945, "p05": 0.09931983798742294, "p95": 0.10953669250011444 }, { "label": "completed_group_6", "mean": 0.09994889795780182, "median": 0.09950269758701324, "p05": 0.09602029621601105, "p95": 0.1042325347661972 }, { "label": "completed_group_7", "mean": 0.0946216881275177, "median": 0.09491042792797089, "p05": 0.09054946899414062, "p95": 0.098105289041996 }, { "label": "current_group_partial", "mean": 0.09481187164783478, "median": 0.09540842473506927, "p05": 0.09017777442932129, "p95": 0.09939083456993103 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.196977138519287, "group": 8, "latest_source_mass": 0.11403195559978485, "layer": 32, "max_source_mass": 0.11424341797828674, "mixer_index": 62, "normalized_entropy": 0.999887385741319, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10978135466575623, "median": 0.10953371971845627, "p05": 0.108054518699646, "p95": 0.11213957518339157 }, { "label": "completed_group_1", "mean": 0.1075383648276329, "median": 0.10733462870121002, "p05": 0.10584845393896103, "p95": 0.10943715274333954 }, { "label": "completed_group_2", "mean": 0.10839901119470596, "median": 0.10859507322311401, "p05": 0.10721821337938309, "p95": 0.10926944017410278 }, { "label": "completed_group_3", "mean": 0.11003260314464569, "median": 0.11011333763599396, "p05": 0.10928525030612946, "p95": 0.11081099510192871 }, { "label": "completed_group_4", "mean": 0.11053797602653503, "median": 0.11058371514081955, "p05": 0.1097012460231781, "p95": 0.11142666637897491 }, { "label": "completed_group_5", "mean": 0.11152469366788864, "median": 0.1117575541138649, "p05": 0.11029430478811264, "p95": 0.1124267652630806 }, { "label": "completed_group_6", "mean": 0.11391061544418335, "median": 0.11399351060390472, "p05": 0.11294781416654587, "p95": 0.11480505764484406 }, { "label": "completed_group_7", "mean": 0.11424341797828674, "median": 0.11410531401634216, "p05": 0.11359494924545288, "p95": 0.11528046429157257 }, { "label": "current_group_partial", "mean": 0.11403195559978485, "median": 0.11387525498867035, "p05": 0.11328872293233871, "p95": 0.11511529237031937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1905651092529297, "group": 8, "latest_source_mass": 0.0946686640381813, "layer": 32, "max_source_mass": 0.1320299208164215, "mixer_index": 63, "normalized_entropy": 0.9969691454610601, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11943811178207397, "median": 0.12088920176029205, "p05": 0.10572259873151779, "p95": 0.1288197934627533 }, { "label": "completed_group_1", "mean": 0.1320299208164215, "median": 0.13276371359825134, "p05": 0.12282373756170273, "p95": 0.13998818397521973 }, { "label": "completed_group_2", "mean": 0.12506362795829773, "median": 0.12487159669399261, "p05": 0.12049838155508041, "p95": 0.12907996773719788 }, { "label": "completed_group_3", "mean": 0.11625386774539948, "median": 0.11589770764112473, "p05": 0.11143262684345245, "p95": 0.12191615998744965 }, { "label": "completed_group_4", "mean": 0.11043961346149445, "median": 0.10992902517318726, "p05": 0.10569487512111664, "p95": 0.11613331735134125 }, { "label": "completed_group_5", "mean": 0.1047937273979187, "median": 0.10408039391040802, "p05": 0.10086895525455475, "p95": 0.11024832725524902 }, { "label": "completed_group_6", "mean": 0.1010652706027031, "median": 0.10061101615428925, "p05": 0.09771151095628738, "p95": 0.10514533519744873 }, { "label": "completed_group_7", "mean": 0.09624720364809036, "median": 0.09628060460090637, "p05": 0.09263256937265396, "p95": 0.09951217472553253 }, { "label": "current_group_partial", "mean": 0.0946686640381813, "median": 0.09522657096385956, "p05": 0.09038722515106201, "p95": 0.09931783378124237 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1935477256774902, "group": null, "latest_source_mass": 0.11779685318470001, "layer": null, "max_source_mass": 0.11986880004405975, "mixer_index": null, "normalized_entropy": 0.9983265926948683, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.10072024166584015, "median": 0.1001858189702034, "p05": 0.09205758571624756, "p95": 0.11032550781965256 }, { "label": "completed_group_1", "mean": 0.09480114281177521, "median": 0.0942176878452301, "p05": 0.09034405648708344, "p95": 0.0996023491024971 }, { "label": "completed_group_2", "mean": 0.10205776989459991, "median": 0.10174158215522766, "p05": 0.09974265098571777, "p95": 0.10526342689990997 }, { "label": "completed_group_3", "mean": 0.11059840023517609, "median": 0.11073200404644012, "p05": 0.1057434156537056, "p95": 0.11464740335941315 }, { "label": "completed_group_4", "mean": 0.1157325878739357, "median": 0.11606095731258392, "p05": 0.11147730052471161, "p95": 0.12005195766687393 }, { "label": "completed_group_5", "mean": 0.11904929578304291, "median": 0.11922121047973633, "p05": 0.11581429839134216, "p95": 0.1222524642944336 }, { "label": "completed_group_6", "mean": 0.11986880004405975, "median": 0.11976201832294464, "p05": 0.11818123608827591, "p95": 0.12199589610099792 }, { "label": "completed_group_7", "mean": 0.11937490850687027, "median": 0.11958470195531845, "p05": 0.11761733889579773, "p95": 0.12075771391391754 }, { "label": "completed_group_8", "mean": 0.11779685318470001, "median": 0.1180996373295784, "p05": 0.11525309085845947, "p95": 0.11991780996322632 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "334f6a2e9305f6b7249f6d6caf2b255325d80c7e8a5aad5effed4c164d06179a", "optimizer_state_before": "334f6a2e9305f6b7249f6d6caf2b255325d80c7e8a5aad5effed4c164d06179a", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "18380f2ab4362bd0b1746896f121e69ee1546f1794cea323af41740b6024651d", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.003087003482505679, 0.0026560353580862284, 0.0032712947577238083, 0.002734958194196224, 0.003094236133620143, 0.0030397549271583557, 0.0035086015705019236, 0.0038121570833027363, 0.0047028325498104095, 0.004038033075630665, 0.004860103130340576, 0.005031581036746502, 0.005196469835937023, 0.005135253071784973, 0.006614624988287687, 0.005866772960871458, 0.006203304044902325, 0.007167549338191748, 0.007609309628605843, 0.008147990331053734, 0.00901782512664795, 0.008936969563364983, 0.009206709451973438, 0.01152731105685234, 0.010577243752777576, 0.011347576975822449, 0.011137762106955051, 0.012936098501086235, 0.013407763093709946, 0.013788831420242786, 0.015265416353940964, 0.0202538650482893 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.47538081591847e-05, "first_to_last_ratio": 3.523320813580708, "imbalance_abs_log_ratio": 1.2594039578247644, "last_quartile_mean": 4.187472256944602e-06, "mean": 8.003732475003744e-06, "normalized": [ 2.2864072450452606, 2.1505052861894143, 2.035100368661317, 1.914613292131888, 1.7534318989662239, 1.6431636846734412, 1.536642318353803, 1.4270637477194381, 1.2842766020276717, 1.1810344805085975, 1.099480510026868, 1.0224360977234737, 0.928328885424462, 0.8663541927168132, 0.8170363190541032, 0.770456622754232, 0.7301180367940027, 0.6940574801022341, 0.6637352609099907, 0.6396770492722799, 0.6174712393642938, 0.5998406679830636, 0.583621712740686, 0.5696275372587853, 0.5553044486825874, 0.5452271173697439, 0.5355955147189773, 0.527231951874486, 0.5148029686402454, 0.5078768736288382, 0.5021984211528733, 0.49728216752990523 ], "peak_layer": 1, "peak_normalized": 2.2864072450452606, "population_cv": 0.5439316214178195, "rest_mean": 8.618577541912802e-06, "spike_contrast": 0.5434271596349021, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.68356911369483e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8299791918252595e-05, 1.7212068996741436e-05, 1.6288398910546675e-05, 1.5324052583309822e-05, 1.403399983246345e-05, 1.3151442544767633e-05, 1.2298874025873374e-05, 1.1421836461522616e-05, 1.0279006346536335e-05, 9.452684025745839e-06, 8.799947863735724e-06, 8.183304998965468e-06, 7.4300960477557965e-06, 6.93406718710321e-06, 6.539340120070847e-06, 6.166528692119755e-06, 5.8436694416741375e-06, 5.5550503930135164e-06, 5.312359462550376e-06, 5.1198039727751166e-06, 4.942074610880809e-06, 4.800964234163985e-06, 4.671152055379935e-06, 4.559146418614546e-06, 4.444508249434875e-06, 4.363851985544898e-06, 4.286763214622624e-06, 4.219823495077435e-06, 4.1203452383342665e-06, 4.064910626766505e-06, 4.019461812276859e-06, 3.980113433499355e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.9807127854583086e-05, "first_to_last_ratio": 3.5566643349459857, "imbalance_abs_log_ratio": 1.2688231210537682, "last_quartile_mean": 1.681551088950073e-05, "mean": 3.247700539077414e-05, "normalized": [ 2.2579241602152, 2.135019037063823, 2.0280333310485936, 1.9173080473526447, 1.758657494805108, 1.6519298761064487, 1.544128685861779, 1.4391773217546358, 1.2962904826236146, 1.192609616061844, 1.110941110824676, 1.032164886192474, 0.9359594784290148, 0.8726072754652816, 0.821823275119541, 0.7740717447349674, 0.7320077127739505, 0.6952135796200956, 0.6640023209515697, 0.639026268143393, 0.6157178693827154, 0.5971561937295294, 0.5802935929104301, 0.5658032894829641, 0.5509069802052773, 0.5405054676570583, 0.530569722062962, 0.5218335672124204, 0.509106468388458, 0.5020150600957286, 0.49619577359580697, 0.49100031012799433 ], "peak_layer": 1, "peak_normalized": 2.2579241602152, "population_cv": 0.5433134428802157, "rest_mean": 3.499111301573942e-05, "spike_contrast": 0.5401607033037161, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.890082421596162e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.333061512326822e-05, 6.933902477612719e-05, 6.586444942513481e-05, 6.226842378964648e-05, 5.7115928939310834e-05, 5.364973549148999e-05, 5.014867565478198e-05, 4.674016963690519e-05, 4.209963299217634e-05, 3.8732388929929584e-05, 3.6080040445085615e-05, 3.3521624573040754e-05, 3.0397161026485264e-05, 2.8339671189314686e-05, 2.669035893632099e-05, 2.5139532226603478e-05, 2.3773418433847837e-05, 2.257845517306123e-05, 2.156480695703067e-05, 2.0753659555339254e-05, 1.9996672563138418e-05, 1.9393844922888093e-05, 1.8846198145183735e-05, 1.8375596482655965e-05, 1.789180896594189e-05, 1.755399898684118e-05, 1.7231315723620355e-05, 1.6947591575444676e-05, 1.6534253518329933e-05, 1.6303945812978782e-05, 1.6114952813950367e-05, 1.5946219718898647e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05303755821660161, "first_to_last_ratio": 3.5566642959691026, "imbalance_abs_log_ratio": 1.2688231100949372, "last_quartile_mean": 0.014912163140252233, "mean": 0.028800932981539518, "normalized": [ 2.257924127684546, 2.135019140736797, 2.0280331746484235, 1.9173079402069289, 1.7586574931696988, 1.651929831377606, 1.5441287257423313, 1.4391773748670262, 1.2962904745959671, 1.192609691122949, 1.1109411677627854, 1.03216488937858, 0.9359594736307031, 0.8726072729963495, 0.8218232514089269, 0.7740717507159982, 0.7320077303494418, 0.695213596207834, 0.6640023013431785, 0.6390262678237983, 0.6157178297675131, 0.5971562020352453, 0.5802936008847488, 0.56580333779038, 0.5509069924034854, 0.5405054586699878, 0.5305697328218089, 0.5218335956665606, 0.5091064501799054, 0.5020150788892077, 0.49619576248172176, 0.4910002826395651 ], "peak_layer": 1, "peak_normalized": 2.257924127684546, "population_cv": 0.5433134335599328, "rest_mean": 0.031030468711698497, "spike_contrast": 0.5401607108938693, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01676144003868103, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06503032147884369, 0.06149054318666458, 0.05840924754738808, 0.05522025749087334, 0.05065097659826279, 0.04757712036371231, 0.0444723479449749, 0.0414496511220932, 0.0373343750834465, 0.034348271787166595, 0.031996142119169235, 0.029727311804890633, 0.026956506073474884, 0.02513190358877182, 0.023669276386499405, 0.02229398861527443, 0.021082505583763123, 0.0200228001922369, 0.01912388578057289, 0.018404552713036537, 0.017733247950673103, 0.017198655754327774, 0.01671299710869789, 0.0162956640124321, 0.015866635367274284, 0.015567061491310596, 0.015280903317034245, 0.015029294416308403, 0.014662740752100945, 0.014458502642810345, 0.014290900900959969, 0.014141266234219074 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.542342338709204e-06, "first_to_last_ratio": 3.9100869045545728, "imbalance_abs_log_ratio": 1.3635595999790664, "last_quartile_mean": 1.1616985631235366e-06, "mean": 2.356269877878958e-06, "normalized": [ 2.626358068851995, 2.359482633900792, 2.1673808318573355, 1.970612227511214, 1.7715197752462144, 1.611066095373113, 1.5253639505266257, 1.390362687339307, 1.2470742362409657, 1.1482697848345025, 1.0649746685773553, 0.9926077747670571, 0.8935238671869759, 0.8352705716146634, 0.7976779264251802, 0.7534193926689576, 0.7091631265992269, 0.6729593670129498, 0.6388931190640394, 0.6146294043377577, 0.5903414687897984, 0.573600345637639, 0.5583270911267058, 0.5429263154370998, 0.5273495286768174, 0.5154150806600781, 0.5059577182706787, 0.4967911022387073, 0.4840893077120886, 0.476822723468287, 0.4716374894346956, 0.4661323186111764 ], "peak_layer": 1, "peak_normalized": 2.626358068851995, "population_cv": 0.6048322607097205, "rest_mean": 2.5489128524390658e-06, "spike_contrast": 0.5162976890304795, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3159978152543771e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.188408406160306e-06, 5.559577857638942e-06, 5.106934167997679e-06, 4.64329423266463e-06, 4.174178684479557e-06, 3.796106511799735e-06, 3.5941691294283373e-06, 3.2760697195044486e-06, 2.938443458333495e-06, 2.7056335056840908e-06, 2.509367732272949e-06, 2.338851800232078e-06, 2.10538337341859e-06, 1.9681228877743706e-06, 1.8795444702845998e-06, 1.7752594203557237e-06, 1.6709797137082205e-06, 1.5856738855291042e-06, 1.5054046116347308e-06, 1.448232751499745e-06, 1.391003820572223e-06, 1.3515572163669276e-06, 1.3155693068256369e-06, 1.279280922972248e-06, 1.2425778095348505e-06, 1.2144570291638956e-06, 1.1921729310415685e-06, 1.1705739098033519e-06, 1.1406450539652724e-06, 1.123523020396533e-06, 1.1113052096334286e-06, 1.0983335414493922e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.8821781547303544e-05, "first_to_last_ratio": 2.980254400081657, "imbalance_abs_log_ratio": 1.0920086660284927, "last_quartile_mean": 1.6381749674110324e-05, "mean": 2.848307332214972e-05, "normalized": [ 2.0810926090025332, 1.969923093544882, 1.8754316812286433, 1.775123579435819, 1.6405766968825957, 1.5475620321943906, 1.4549157242056538, 1.367878426084163, 1.2540203154379537, 1.1689725777559958, 1.0996704285612149, 1.0350754283963208, 0.960189941905928, 0.9068543441283098, 0.8632891395831829, 0.821927178126067, 0.7877167521462786, 0.7537929393542872, 0.7245378305968498, 0.700636091646331, 0.6787484361888465, 0.6601340974768426, 0.6429525692553493, 0.6278595238949346, 0.6122509835140441, 0.6009361488079162, 0.5899975897039497, 0.5801705507723549, 0.5656037931304417, 0.5573685175022867, 0.5505152181183216, 0.5442757614173119 ], "peak_layer": 1, "peak_normalized": 2.0810926090025332, "population_cv": 0.46429465779931894, "rest_mean": 3.0358793824066027e-05, "spike_contrast": 0.6045754886760992, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.835418261180166e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.927591337240301e-05, 5.6109463912434876e-05, 5.341805808711797e-05, 5.05609750689473e-05, 4.672866634791717e-05, 4.407932283356786e-05, 4.14404712500982e-05, 3.896138150594197e-05, 3.571835259208456e-05, 3.32959316438064e-05, 3.132199344690889e-05, 2.9482129320967942e-05, 2.734916051849723e-05, 2.5829998776316643e-05, 2.4589127860963345e-05, 2.341101208003238e-05, 2.243659400846809e-05, 2.147033956134692e-05, 2.063706415356137e-05, 1.9956269170506857e-05, 1.9332841475261375e-05, 1.880264790088404e-05, 1.831326517276466e-05, 1.7883368855109438e-05, 1.7438789654988796e-05, 1.7116508388426155e-05, 1.6804944607429206e-05, 1.6525040337000974e-05, 1.6110134311020374e-05, 1.5875568351475522e-05, 1.5680365322623402e-05, 1.5502646419918165e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.9543975162814604e-05, "first_to_last_ratio": 2.361814271071357, "imbalance_abs_log_ratio": 0.8594300826222705, "last_quartile_mean": 1.6743050309742102e-05, "mean": 2.5007048577663227e-05, "normalized": [ 1.912859565038877, 1.8105236978529957, 1.7273211150778174, 1.6308285226915125, 1.5141405100255678, 1.4329864048795495, 1.3476327823895244, 1.2742127313731153, 1.1867378921860672, 1.112686902495014, 1.0470750994574345, 0.9973623109170946, 0.9478955259042676, 0.9058401245160093, 0.8736388301292974, 0.8429558183060917, 0.8275675048043188, 0.803939236955949, 0.7837442531107164, 0.7644177737357688, 0.7470428108125197, 0.7311443054887553, 0.7174787440702515, 0.7037016009216458, 0.6932620899019373, 0.6852674844312424, 0.6794925843554572, 0.6746814041897898, 0.6624963737585712, 0.6587300903027569, 0.6539322941255763, 0.6484036157945076 ], "peak_layer": 1, "peak_normalized": 1.912859565038877, "population_cv": 0.375701127391813, "rest_mean": 2.631053676951625e-05, "spike_contrast": 0.6829283833720613, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7968212341656907e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.783497206517495e-05, 4.527585406322032e-05, 4.3195203033974394e-05, 4.078220808878541e-05, 3.786418528761715e-05, 3.583476063795388e-05, 3.370031845406629e-05, 3.186429967172444e-05, 2.967681211885065e-05, 2.782501542242244e-05, 2.618425787659362e-05, 2.494108775863424e-05, 2.370406946283765e-05, 2.265238799736835e-05, 2.1847128664376214e-05, 2.107983709720429e-05, 2.0695020793937147e-05, 2.0104147552046925e-05, 1.9599130610004067e-05, 1.9115832401439548e-05, 1.868133585958276e-05, 1.8283761164639145e-05, 1.794202580640558e-05, 1.759750011842698e-05, 1.7336438759230077e-05, 1.713651727186516e-05, 1.6992104065138847e-05, 1.6871790649020113e-05, 1.6567079001106322e-05, 1.6472895367769524e-05, 1.6352916645701043e-05, 1.6214660718105733e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011525819263624726, "first_to_last_ratio": 5.424369947393303, "imbalance_abs_log_ratio": 1.6909017539525537, "last_quartile_mean": 2.12482175356854e-05, "mean": 5.662517071414186e-05, "normalized": [ 2.5055056176095984, 2.3706297508444365, 2.241830077392606, 2.124145849270733, 1.9443711953183194, 1.8164591538959356, 1.7002866499817122, 1.5804407783115777, 1.4040231414989057, 1.2897275398797217, 1.1905083194532244, 1.0938772119880942, 0.9633505879752084, 0.8774870943542636, 0.8097317824087507, 0.7409024588364089, 0.6759871710557561, 0.6280736730006866, 0.5867444951930028, 0.5522911890332279, 0.5129096169838441, 0.48683288837526706, 0.4612191855365385, 0.4407179437979303, 0.4173472207950482, 0.4025342509602604, 0.3881747341410948, 0.376294400049804, 0.36321827476366064, 0.3558481534629913, 0.35107774164810374, 0.34745185218328734 ], "peak_layer": 1, "peak_normalized": 2.5055056176095984, "population_cv": 0.6753260599958857, "rest_mean": 6.224778412110431e-05, "spike_contrast": 0.4219115376934434, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.626305831654463e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014187468332238495, 0.0001342373143415898, 0.00012694401084445417, 0.00012028012133669108, 0.00011010035086655989, 0.00010285730968462303, 9.627902181819081e-05, 8.949272887548432e-05, 7.950305007398129e-05, 7.303104212041944e-05, 6.741273682564497e-05, 6.194098386913538e-05, 5.4549891501665115e-05, 4.968785651726648e-05, 4.5851200411561877e-05, 4.195372821413912e-05, 3.8277888961602e-05, 3.556477895472199e-05, 3.3224507205886766e-05, 3.127358286292292e-05, 2.9043594622635283e-05, 2.7566995413508266e-05, 2.611661511764396e-05, 2.495572880434338e-05, 2.363235762459226e-05, 2.2793570678913966e-05, 2.1980460587656125e-05, 2.1307734641595744e-05, 2.0567296814988367e-05, 2.0149962438154034e-05, 1.9879837054759264e-05, 1.967452044482343e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "28d398d2ea80931917268ed413eca333d2f81168391c83a3adc104502478e55b", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0049620093777775764, 0.004914142191410065, 0.00555405905470252, 0.005823355168104172, 0.006141432095319033, 0.006136075593531132, 0.006357426289469004, 0.0064498125575482845, 0.006525229662656784, 0.007023691199719906, 0.006682637147605419, 0.006716069765388966, 0.006977807730436325, 0.006669311318546534, 0.006923797074705362, 0.007155857048928738, 0.007240671198815107, 0.007543859537690878, 0.007281317841261625, 0.007711030542850494, 0.00788753479719162, 0.007908415980637074, 0.008078642189502716, 0.008181036449968815, 0.007894126698374748, 0.008452145382761955, 0.008351319469511509, 0.008498509414494038, 0.009082214906811714, 0.009139914996922016, 0.008129997178912163, 0.00888112559914589 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3918628951614664e-05, "first_to_last_ratio": 3.3693491917247846, "imbalance_abs_log_ratio": 1.2147196075319966, "last_quartile_mean": 4.130954721404123e-06, "mean": 7.606227299561397e-06, "normalized": [ 2.2824447028233354, 2.156224576681686, 2.02967801794237, 1.9096018794072827, 1.736504358227357, 1.6206331020107865, 1.508932291267265, 1.395174357296542, 1.2454912604425945, 1.159982779559482, 1.0770364956371037, 0.9985646101779827, 0.9126193313903731, 0.8598667474559943, 0.8109240384824798, 0.7673321935183389, 0.7307233755519057, 0.6989382867571854, 0.6733294103254632, 0.6494885311058664, 0.6312572107399844, 0.6141046126011543, 0.5996497467158993, 0.586684752686662, 0.5738065084191857, 0.5636248604676733, 0.5548170991634841, 0.5473936870898765, 0.5344900934778194, 0.5284722547322606, 0.5232893301307341, 0.5189194977138724 ], "peak_layer": 1, "peak_normalized": 2.2824447028233354, "population_cv": 0.5333395699341017, "rest_mean": 8.168101329728313e-06, "spike_contrast": 0.5597515691951044, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.572107536660042e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.7360793208354153e-05, 1.6400734239141457e-05, 1.543819234939292e-05, 1.4524865946441423e-05, 1.3208246855356265e-05, 1.2326903743087314e-05, 1.14772819870268e-05, 1.0612013284116983e-05, 9.473489626543596e-06, 8.823092684906442e-06, 8.192184395738877e-06, 7.595309398311656e-06, 6.941590072528925e-06, 6.54034192848485e-06, 6.168072559376014e-06, 5.836503078171518e-06, 5.55804808755056e-06, 5.316283477441175e-06, 5.121496542415116e-06, 4.9401573960494716e-06, 4.8014858293754514e-06, 4.671019269153476e-06, 4.56107227364555e-06, 4.462457582121715e-06, 4.364502729004016e-06, 4.2870588004007e-06, 4.220064965920756e-06, 4.163600806350587e-06, 4.065453140356112e-06, 4.019680091005284e-06, 3.980257588409586e-06, 3.947019649785943e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.6411598961858544e-05, "first_to_last_ratio": 3.4027524402802842, "imbalance_abs_log_ratio": 1.2245846453792368, "last_quartile_mean": 1.6578225995544926e-05, "mean": 3.084055111912676e-05, "normalized": [ 2.25484903989412, 2.1412529403387204, 2.024538426739007, 1.9138206908201378, 1.744005847409706, 1.6283901394849352, 1.51851794721373, 1.4077224750694408, 1.2571797277143097, 1.1712619710004104, 1.087585943008638, 1.0075084298767483, 0.9193230520257301, 0.8658498478469501, 0.8153148724555359, 0.7704877885294069, 0.7322822621292467, 0.6994579345140977, 0.6731224584255281, 0.6484140168699147, 0.6289011036610557, 0.6111628530369178, 0.595980107097907, 0.5826992445616984, 0.5692429152040753, 0.5587822615864424, 0.549587741163871, 0.541809636108114, 0.5286857205338498, 0.5225713358473244, 0.5170881598743383, 0.5126031099580918 ], "peak_layer": 1, "peak_normalized": 2.25484903989412, "population_cv": 0.5330474913622731, "rest_mean": 3.3138759034853945e-05, "spike_contrast": 0.5561532450510837, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.843022837419994e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.95407870807685e-05, 6.603742076549679e-05, 6.243788084248081e-05, 5.902328484808095e-05, 5.378610148909502e-05, 5.0220449338667095e-05, 4.683193037635647e-05, 4.3414936953922734e-05, 3.8772115658503026e-05, 3.612236469052732e-05, 3.3541749871801585e-05, 3.107211523456499e-05, 2.8352429580991156e-05, 2.670328649401199e-05, 2.514476000214927e-05, 2.37622680288041e-05, 2.2583988538826816e-05, 2.1571668185060844e-05, 2.0759467588504776e-05, 1.9997445633634925e-05, 1.9395656636334024e-05, 1.8848599211196415e-05, 1.838035495893564e-05, 1.7970765838981606e-05, 1.7555765225552022e-05, 1.7233152902917936e-05, 1.694958882580977e-05, 1.6709707779227756e-05, 1.6304958990076557e-05, 1.6116387996589765e-05, 1.594728382769972e-05, 1.5808962416485883e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05002636928111315, "first_to_last_ratio": 3.4027524331610692, "imbalance_abs_log_ratio": 1.2245846432870438, "last_quartile_mean": 0.014701736392453313, "mean": 0.027349708892870694, "normalized": [ 2.2548490514458424, 2.1412530488236308, 2.024538393384759, 1.9138206338036392, 1.744005792683758, 1.6283900755277658, 1.5185179045510238, 1.4077225055522984, 1.257179674815174, 1.1712619883598294, 1.087585969825964, 1.0075084701933625, 0.9193230998011659, 0.8658498362871361, 0.8153148842057143, 0.7704877914133448, 0.7322822512666904, 0.6994579400171268, 0.6731224452510738, 0.6484140411218751, 0.6289011442302842, 0.6111628528689982, 0.5959801242324752, 0.5826992208035262, 0.5692429135850658, 0.5587822620006379, 0.5495877458659506, 0.5418096383169221, 0.5286857163419914, 0.5225713057341705, 0.5170881921276606, 0.5126030855611432 ], "peak_layer": 1, "peak_normalized": 2.2548490514458424, "population_cv": 0.5330474880442257, "rest_mean": 0.02938778260377822, "spike_contrast": 0.5561532516532492, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01634411085397005, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06166946515440941, 0.05856264755129814, 0.05537053570151329, 0.05234243720769882, 0.04769805073738098, 0.04453599452972412, 0.041531022638082504, 0.03850080072879791, 0.03438349813222885, 0.03203367441892624, 0.029745159670710564, 0.027555063366889954, 0.025143219158053398, 0.02368074096739292, 0.022298624739050865, 0.021072616800665855, 0.020027706399559975, 0.01912997104227543, 0.01840970292687416, 0.01773393526673317, 0.01720026321709156, 0.016715126112103462, 0.016299882903695107, 0.015936654061079025, 0.0155686279758811, 0.015282532200217247, 0.015031064860522747, 0.014818335883319378, 0.014459400437772274, 0.014292173087596893, 0.014142211526632309, 0.014019545167684555 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.3751225575761055e-06, "first_to_last_ratio": 3.8220970734227353, "imbalance_abs_log_ratio": 1.3407992440886183, "last_quartile_mean": 1.1446916374779903e-06, "mean": 2.2650369189136654e-06, "normalized": [ 2.6260982509869395, 2.400216776938017, 2.1919812201941786, 1.9790918569249247, 1.7559460501631168, 1.6114596120900806, 1.5126018597778093, 1.3753240033827305, 1.2117365805847247, 1.134432397143863, 1.0401833788995027, 0.9674957594857626, 0.875746003990826, 0.8237594917355141, 0.7832661047995432, 0.740719511828896, 0.7019638739171157, 0.6704099413729608, 0.6414011471084737, 0.6173397326477368, 0.5968863232373988, 0.5799016861412486, 0.5669030002035256, 0.5521403649934858, 0.5377798684577518, 0.5258530838590796, 0.5176403099124206, 0.5093490356054753, 0.496021692149111, 0.49050001495252427, 0.4851144585852832, 0.4807366079299788 ], "peak_layer": 1, "peak_normalized": 2.6260982509869395, "population_cv": 0.6051676860135061, "rest_mean": 2.4467758268858434e-06, "spike_contrast": 0.5246278803962507, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2836468158639037e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.948209491180023e-06, 5.436579613160575e-06, 4.9649183893052395e-06, 4.482716121856356e-06, 3.977282631240087e-06, 3.6500155147223268e-06, 3.426099056014209e-06, 3.1151596431300277e-06, 2.744628091022605e-06, 2.569531261542579e-06, 2.3560537556477357e-06, 2.1914136141276686e-06, 1.983597030630335e-06, 1.865845661086496e-06, 1.7741266447046655e-06, 1.677757040852157e-06, 1.5899740901659243e-06, 1.5185032680165023e-06, 1.452797278034268e-06, 1.3982972859594156e-06, 1.351969558527344e-06, 1.313498728450213e-06, 1.2840562249039067e-06, 1.2506183111327118e-06, 1.2180912563053425e-06, 1.191076648865419e-06, 1.1724744126695441e-06, 1.1536943702594726e-06, 1.123507445299765e-06, 1.1110006425951724e-06, 1.0988021585944807e-06, 1.088886165234726e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.6260539875220275e-05, "first_to_last_ratio": 2.865320598644344, "imbalance_abs_log_ratio": 1.0526802457017466, "last_quartile_mean": 1.6144978644661023e-05, "mean": 2.7227252871853125e-05, "normalized": [ 2.069229242257896, 1.9687734840713744, 1.8639501074620208, 1.7653990360295826, 1.6236824582203917, 1.5253078926597279, 1.434250504279973, 1.341829571296541, 1.2243737990511798, 1.1518364329038495, 1.0836884644807132, 1.0178401497737837, 0.9492199847186634, 0.9036392473516051, 0.8600860560490738, 0.8201864427615304, 0.7888244492232437, 0.7582210568106187, 0.7331773989358682, 0.7093267256759077, 0.6906736225590872, 0.672715786894029, 0.6570059626140338, 0.6429919843026688, 0.6287389013300317, 0.6172847926207785, 0.6070179898741197, 0.5981761279825236, 0.5831187474359685, 0.5759696583600498, 0.5694268489156044, 0.564037073097561 ], "peak_layer": 1, "peak_normalized": 2.069229242257896, "population_cv": 0.4529330483375927, "rest_mean": 2.894950139980884e-05, "spike_contrast": 0.6192545623950071, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7927110820892268e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.633942782878876e-05, 5.360429349821061e-05, 5.075024091638625e-05, 4.806696597370319e-05, 4.42084128735587e-05, 4.152994370087981e-05, 3.905070116161369e-05, 3.653433304862119e-05, 3.333633503643796e-05, 3.1361341825686395e-05, 2.9505859856726602e-05, 2.7712991141015664e-05, 2.5844652554951608e-05, 2.4603614292573184e-05, 2.3417780539602973e-05, 2.2331423679133877e-05, 2.147752275050152e-05, 2.0644276446546428e-05, 1.996240644075442e-05, 1.9313018128741533e-05, 1.8805145373335108e-05, 1.8316202840651385e-05, 1.7888467482407577e-05, 1.7506905351183377e-05, 1.7118833056883886e-05, 1.6806969142635353e-05, 1.6527432308066636e-05, 1.628669269848615e-05, 1.587672159075737e-05, 1.5682071534683928e-05, 1.5503928807447664e-05, 1.53571800183272e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.756828982659499e-05, "first_to_last_ratio": 2.2614511370462873, "imbalance_abs_log_ratio": 0.8160067032669138, "last_quartile_mean": 1.661247029005608e-05, "mean": 2.4095109154131933e-05, "normalized": [ 1.889963326736216, 1.799730688213, 1.7036000060988148, 1.610459564444366, 1.4904922646790375, 1.4035903358019606, 1.3280490559101101, 1.2474476942829602, 1.158111146500897, 1.0893113074839385, 1.0361619537088214, 0.9872898290164548, 0.9417149709880597, 0.9072412957378696, 0.8752606702463853, 0.8478439926779969, 0.8346944083690043, 0.8136049467473944, 0.7934463022141092, 0.7733479003439614, 0.7587235786610493, 0.7446988149112661, 0.7306767688756092, 0.718906877438168, 0.7112476796011702, 0.7050319654256372, 0.7000888963818952, 0.6958494133286793, 0.6835587021757187, 0.6787900952804858, 0.67292504171933, 0.6681405059996335 ], "peak_layer": 1, "peak_normalized": 1.889963326736216, "population_cv": 0.3606025174079256, "rest_mean": 2.5287144429360826e-05, "spike_contrast": 0.6983041804986536, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7658118667895906e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.553887265501544e-05, 4.336470738053322e-05, 4.104842810193077e-05, 3.880419899360277e-05, 3.591357381083071e-05, 3.3819662348832935e-05, 3.199948696419597e-05, 3.0057388357818127e-05, 2.7904814487555996e-05, 2.6247074856655672e-05, 2.496643537597265e-05, 2.378885619691573e-05, 2.2690725018037483e-05, 2.1860078049940057e-05, 2.108950138790533e-05, 2.042889354925137e-05, 2.0112052879994735e-05, 1.9603900000220165e-05, 1.9118175259791315e-05, 1.8633902072906494e-05, 1.828152744565159e-05, 1.794359923223965e-05, 1.7605736502446234e-05, 1.7322139683528803e-05, 1.713759047561325e-05, 1.69878221640829e-05, 1.6868718375917524e-05, 1.6766567568993196e-05, 1.6470421542180702e-05, 1.635552143852692e-05, 1.621420233277604e-05, 1.609891842235811e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010847938028746285, "first_to_last_ratio": 5.225746258439172, "imbalance_abs_log_ratio": 1.653597612234075, "last_quartile_mean": 2.075863903883146e-05, "mean": 5.315361454449885e-05, "normalized": [ 2.5242915739433354, 2.39631132323169, 2.26527797895308, 2.1515636583598874, 1.938662398543429, 1.8109880868616515, 1.684831424407678, 1.5549978812909424, 1.3759063748543887, 1.26834355048635, 1.1636757169651435, 1.0626666952243982, 0.9347756253977458, 0.8624265202529392, 0.7881968824941187, 0.7298512980179852, 0.6687342618334785, 0.625193525578969, 0.5885755667414656, 0.5509759337416824, 0.518584792950607, 0.49089751607579774, 0.469366663303134, 0.45058077490001597, 0.42852582187652194, 0.4135415510225047, 0.4009779970762889, 0.3909110972534533, 0.3790735154078651, 0.37379596447854513, 0.37033105096913865, 0.36716697750576865 ], "peak_layer": 1, "peak_normalized": 2.5242915739433354, "population_cv": 0.6748031786716517, "rest_mean": 5.8354881629283125e-05, "spike_contrast": 0.4295574181078448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5066772286663765e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013417522131931037, 0.00012737260840367526, 0.0001204077125294134, 0.00011436338536441326, 0.00010304691386409104, 9.626056271372363e-05, 8.955488010542467e-05, 8.265375799965113e-05, 7.313439709832892e-05, 6.741704419255257e-05, 6.185357051435858e-05, 5.64845759072341e-05, 4.9686703277984634e-05, 4.584108683047816e-05, 4.189551327726804e-05, 3.879423456965014e-05, 3.5545643186196685e-05, 3.32312956743408e-05, 3.128491880488582e-05, 2.9286362405400723e-05, 2.7564656193135306e-05, 2.609297735034488e-05, 2.4948534701252356e-05, 2.394999683019705e-05, 2.277769635838922e-05, 2.1981228201184422e-05, 2.1313429897418246e-05, 2.077833778457716e-05, 2.0149127522017807e-05, 1.9868606614181772e-05, 1.9684433937072754e-05, 1.9516251995810308e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "1ff52033329506a86a2df82471984832a6abc7bac6dc80a52bb5d468ca4544c7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.003087003482505679, 0.006683253683149815, 0.010629965923726559, 0.01484723947942257, 0.003094236133620143, 0.007696541491895914, 0.012912816368043423, 0.018475037068128586, 0.0047028325498104095, 0.0095678074285388, 0.01631263643503189, 0.023062339052557945, 0.005196469835937023, 0.011978921480476856, 0.01994149386882782, 0.027569252997636795, 0.006203304044902325, 0.014378104358911514, 0.024205319583415985, 0.03444278985261917, 0.00901782512664795, 0.01929495297372341, 0.030904190614819527, 0.04446368291974068, 0.010577243752777576, 0.023803669959306717, 0.03807948902249336, 0.05412018671631813, 0.013407763093709946, 0.030261604115366936, 0.04958754777908325, 0.07224015146493912 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4753848404325254e-05, "first_to_last_ratio": 3.523327698234082, "imbalance_abs_log_ratio": 1.2594059118468395, "last_quartile_mean": 4.187475497019477e-06, "mean": 8.003761109875995e-06, "normalized": [ 2.2863863380761895, 2.1504887289697443, 2.035107178272848, 1.9146241690765207, 1.7534335800921985, 1.6431624649344594, 1.536645343249951, 1.4270675055525637, 1.2842963248514263, 1.1810293460813548, 1.0994791900092338, 1.022432212510507, 0.9283341434863113, 0.8663573430191046, 0.817050781874415, 0.7704539231259797, 0.7301217881386737, 0.6940567583066531, 0.6637388520338441, 0.6396683404264909, 0.6174723824373465, 0.5998404537149282, 0.5836225791994625, 0.569622544846661, 0.5553030869663969, 0.5452192009689635, 0.5355922917469447, 0.5272338155115143, 0.5148020927255124, 0.507880056479642, 0.5021977039665544, 0.4972794793476046 ], "peak_layer": 1, "peak_normalized": 2.2863863380761895, "population_cv": 0.5439307229766922, "rest_mean": 8.61860972791975e-06, "spike_contrast": 0.5434262276974213, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.683578572439728e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8299690054845996e-05, 1.7211998056154698e-05, 1.6288511687889695e-05, 1.53241944644833e-05, 1.4034063497092575e-05, 1.3151479834050406e-05, 1.2298942237976007e-05, 1.1421907402109355e-05, 1.0279200978402514e-05, 9.452676749788225e-06, 8.799968782113865e-06, 8.183303179976065e-06, 7.43016471460578e-06, 6.9341172093118075e-06, 6.539479272760218e-06, 6.166529146867106e-06, 5.843720373377437e-06, 5.555064490181394e-06, 5.312407211022219e-06, 5.119752586324466e-06, 4.942101440974511e-06, 4.800979695573915e-06, 4.671175702242181e-06, 4.5591227717523e-06, 4.444513251655735e-06, 4.3638042370730545e-06, 4.286752755433554e-06, 4.2198535084025934e-06, 4.1203529690392315e-06, 4.064950644533383e-06, 4.019470452476526e-06, 3.980106157541741e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.980720561638009e-05, "first_to_last_ratio": 3.556659196689387, "imbalance_abs_log_ratio": 1.2688216763685727, "last_quartile_mean": 1.6815557046356844e-05, "mean": 3.2477058425683936e-05, "normalized": [ 2.257907031008062, 2.1350155505860586, 2.0280560071945923, 1.9172997636195261, 1.7586560791437353, 1.6519288587646246, 1.5441298608627463, 1.4391799003264687, 1.2963020318458602, 1.1925989312182288, 1.1109307833791746, 1.032159280080384, 0.9359660712347485, 0.8726045623068648, 0.8218273098956514, 0.7740706487041483, 0.732006853458854, 0.6952173170733782, 0.6640029168915916, 0.6390278010042643, 0.6157213445930976, 0.597151914077787, 0.5802953336984127, 0.5658058940601672, 0.5509085449477206, 0.5405036888786843, 0.530568183543385, 0.5218326590517988, 0.5091077653403295, 0.502017712830421, 0.49619905192622005, 0.49100034845301377 ], "peak_layer": 1, "peak_normalized": 2.257907031008062, "population_cv": 0.5433125364543616, "rest_mean": 3.4991164014812695e-05, "spike_contrast": 0.5401617458735403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8900888244388624e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.333017856581137e-05, 6.933902477612719e-05, 6.586529343621805e-05, 6.226825644262135e-05, 5.7115976233035326e-05, 5.36497900611721e-05, 5.0148795708082616e-05, 4.6740329707972705e-05, 4.2100076825590804e-05, 3.873210516758263e-05, 3.6079763958696276e-05, 3.3521497243782505e-05, 3.039742477994878e-05, 2.8339629352558404e-05, 2.669053355930373e-05, 2.513953768357169e-05, 2.377342934778426e-05, 2.2578613425139338e-05, 2.1564861526712775e-05, 2.0753743228851818e-05, 1.99968180822907e-05, 1.9393737602513283e-05, 1.8846285456675105e-05, 1.8375711078988388e-05, 1.7891889001475647e-05, 1.7553969883010723e-05, 1.7231293895747513e-05, 1.6947589756455272e-05, 1.6534322639927268e-05, 1.63040585903218e-05, 1.6115085600176826e-05, 1.59462470037397e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.053037628065794706, "first_to_last_ratio": 3.5566592064140465, "imbalance_abs_log_ratio": 1.2688216791027844, "last_quartile_mean": 0.014912204118445516, "mean": 0.02880098018795252, "normalized": [ 2.2579069748415974, 2.135015511975382, 2.028055978471918, 1.9172997531400104, 1.758656162783437, 1.651928934615846, 1.5441298165159716, 1.4391799311213385, 1.2963020605748452, 1.192598940844177, 1.110930680689234, 1.0321593172240922, 0.9359660883361088, 0.8726045492848026, 0.8218273369249041, 0.7740706759881554, 0.7320068539128299, 0.6952173071872988, 0.6640028945037896, 0.6390278073432092, 0.6157213476800849, 0.5971519249410595, 0.5802953660162322, 0.5658058380749674, 0.5509085470093485, 0.5405036996661936, 0.5305681841221567, 0.5218326756792125, 0.5091077499343003, 0.5020176837263095, 0.4961990559232735, 0.49100035094791344 ], "peak_layer": 1, "peak_normalized": 2.2579069748415974, "population_cv": 0.5433125339470698, "rest_mean": 0.03103051417403751, "spike_contrast": 0.5401617443102993, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.016761496663093567, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06502993404865265, 0.06149053946137428, 0.058410000056028366, 0.0552201122045517, 0.05065102130174637, 0.04757717251777649, 0.044472452253103256, 0.04144979268312454, 0.03733476996421814, 0.0343480184674263, 0.03199589252471924, 0.02972720004618168, 0.026956740766763687, 0.025131866335868835, 0.02366943284869194, 0.022293994203209877, 0.02108251489698887, 0.020022939890623093, 0.01912393420934677, 0.018404627218842506, 0.01773337833583355, 0.017198560759425163, 0.01671307533979416, 0.016295762732625008, 0.015866706147789955, 0.015567036345601082, 0.015280883759260178, 0.015029292553663254, 0.01466280221939087, 0.014458601363003254, 0.014291019178926945, 0.014141291379928589 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.542383834404973e-06, "first_to_last_ratio": 3.910089142398143, "imbalance_abs_log_ratio": 1.363560172304691, "last_quartile_mean": 1.1617085107218372e-06, "mean": 2.3562732494042393e-06, "normalized": [ 2.6263801721086546, 2.3595531746074276, 2.1673972230391225, 1.9706080568572362, 1.7715820865194194, 1.611051245513112, 1.5253714176420914, 1.390321713055932, 1.2471349819967512, 1.1481946109719883, 1.064921325760361, 0.9926023015917457, 0.8935298259557028, 0.835274104810251, 0.7976860970261026, 0.7533694870596088, 0.7091585414827071, 0.6729707074821024, 0.6388703482831869, 0.6146359069144856, 0.590298599573816, 0.5736151091802485, 0.5583379201398495, 0.5429116429881082, 0.5273414403230551, 0.5154169485904055, 0.5059683327262104, 0.4967915493601712, 0.4840866850990917, 0.476824743116495, 0.4716594914143145, 0.46613420881024553 ], "peak_layer": 1, "peak_normalized": 2.6263801721086546, "population_cv": 0.60484183936143, "rest_mean": 2.548919993656724e-06, "spike_contrast": 0.5162895790043583, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3159808304408217e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.188469342305325e-06, 5.559752025874332e-06, 5.106980097480118e-06, 4.643291049433174e-06, 4.174331479589455e-06, 3.7960769532219274e-06, 3.5941918667958817e-06, 3.2759778605395695e-06, 2.9385907964751823e-06, 2.705460246943403e-06, 2.509245632609236e-06, 2.3388422505377093e-06, 2.1054004264442483e-06, 1.9681340290844673e-06, 1.87956641184428e-06, 1.7751443692759494e-06, 1.6709713008822291e-06, 1.5857028756727232e-06, 1.505353111497243e-06, 1.4482501455859165e-06, 1.3909047993365675e-06, 1.3515939372155117e-06, 1.3155967053535278e-06, 1.2792481811629841e-06, 1.2425605291355168e-06, 1.2144631682531326e-06, 1.192199647448433e-06, 1.1705766382874572e-06, 1.1406405064917635e-06, 1.1235293868594454e-06, 1.1113586424471578e-06, 1.0983395668517915e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.882185112364823e-05, "first_to_last_ratio": 2.9802495883434017, "imbalance_abs_log_ratio": 1.0920070514877542, "last_quartile_mean": 1.6381799468945246e-05, "mean": 2.8483126868650288e-05, "normalized": [ 2.0810870362663727, 1.9699227110363493, 1.8754345417339926, 1.775115005630423, 1.6405724631848402, 1.547564615006236, 1.4549152880868532, 1.3678859447538592, 1.2540251105018627, 1.1689693583685432, 1.0996615918776176, 1.0350748874830915, 0.9601920962526763, 0.9068520006787653, 0.8632928171993581, 0.8219285067442073, 0.7877145049439734, 0.7537945876465386, 0.7245395338868796, 0.7006371373881553, 0.6787509919040607, 0.6601298549520191, 0.6429542981965809, 0.627861217348151, 0.6122530256203341, 0.600934635913477, 0.589995714201328, 0.5801691407779421, 0.5656053481725521, 0.5573713652661699, 0.5505186535239659, 0.5442760154528234 ], "peak_layer": 1, "peak_normalized": 2.0810870362663727, "population_cv": 0.46429372489568066, "rest_mean": 3.0358846911608622e-05, "spike_contrast": 0.6045762768959777, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.835423863667529e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.927586607867852e-05, 5.610955849988386e-05, 5.3418239986058325e-05, 5.056082591181621e-05, 4.6728633606107906e-05, 4.407947926665656e-05, 4.1440536733716726e-05, 3.896166890626773e-05, 3.5718556318897754e-05, 3.329590253997594e-05, 3.1321800634032115e-05, 2.948216933873482e-05, 2.734927329584025e-05, 2.5829980586422607e-05, 2.458927883708384e-05, 2.3411093934555538e-05, 2.243657218059525e-05, 2.147042687283829e-05, 2.063715146505274e-05, 1.995633647311479e-05, 1.9332950614625588e-05, 1.8802562408382073e-05, 1.8313348846277222e-05, 1.7883450709632598e-05, 1.7438880604458973e-05, 1.7116497474489734e-05, 1.6804922779556364e-05, 1.6525031242053956e-05, 1.611020888958592e-05, 1.5875679309829138e-05, 1.568049265188165e-05, 1.5502682799706236e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.9545781419292325e-05, "first_to_last_ratio": 2.3619145181348626, "imbalance_abs_log_ratio": 0.8594725266608655, "last_quartile_mean": 1.6743104424676858e-05, "mean": 2.5006789769577153e-05, "normalized": [ 1.9129420639284915, 1.8107688022127852, 1.7275839797255208, 1.6309198865482473, 1.5143239186683517, 1.4329543911459017, 1.3474859747283041, 1.2742350840712526, 1.1868783419088809, 1.1125469739554936, 1.0468013052560716, 0.9972033675681935, 0.9479134830208824, 0.9058158937226485, 0.8735998635837403, 0.8429661427640184, 0.8275111858046021, 0.8037572699643292, 0.7836623125837494, 0.7643368697661114, 0.747091058421628, 0.731126704498011, 0.7175431249051513, 0.7036933175686139, 0.6932477338397589, 0.6853092007703565, 0.6793487543844439, 0.6746621277382263, 0.6626876984583827, 0.658797136409654, 0.6539462632483479, 0.6483397688298493 ], "peak_layer": 1, "peak_normalized": 1.9129420639284915, "population_cv": 0.37575052857690755, "rest_mean": 2.631019742693752e-05, "spike_contrast": 0.6829438840103251, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.796838841983117e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.783654003404081e-05, 4.528151475824416e-05, 4.320132939028554e-05, 4.078407073393464e-05, 3.7868379877181724e-05, 3.5833589208777994e-05, 3.3696298487484455e-05, 3.1864528864389285e-05, 2.9680017178179696e-05, 2.7821228286484256e-05, 2.6177140171057545e-05, 2.4936854970292188e-05, 2.370427318965085e-05, 2.2651547624263912e-05, 2.1845928131369874e-05, 2.107987711497117e-05, 2.0693398255389184e-05, 2.009938907576725e-05, 1.9596878701122478e-05, 1.9113611415377818e-05, 1.8682349036680534e-05, 1.8283131794305518e-05, 1.7943450075108558e-05, 1.759711085469462e-05, 1.7335900338366628e-05, 1.7137383110821247e-05, 1.6988331481115893e-05, 1.6871133993845433e-05, 1.6571691958233714e-05, 1.647440149099566e-05, 1.6353096725651994e-05, 1.6212896298384294e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.000115249800728634, "first_to_last_ratio": 5.424078025809064, "imbalance_abs_log_ratio": 1.6908479358239088, "last_quartile_mean": 2.1247813947411487e-05, "mean": 5.662298849529179e-05, "normalized": [ 2.5049859005845536, 2.3700562634212203, 2.2418900056421593, 2.1239131488521026, 1.9444163187103436, 1.8169448512252133, 1.7002382001047527, 1.580666293949324, 1.404181335399843, 1.2897692783756878, 1.1905349262448084, 1.0938543492760489, 0.9632283128394009, 0.8775672358607169, 0.8099566359606734, 0.7410335544511957, 0.6760490100055481, 0.6280269475546286, 0.5865953700288571, 0.5523741532147923, 0.5129543450327941, 0.48678309680551346, 0.4611919541417525, 0.44078321206045507, 0.41741618217688703, 0.4024211697379105, 0.38818779883207655, 0.3763044690378674, 0.36334143229992716, 0.35587172989953436, 0.3510644158471216, 0.34739810242629 ], "peak_layer": 1, "peak_normalized": 2.5049859005845536, "population_cv": 0.6752679783118715, "rest_mean": 6.224517144599101e-05, "spike_contrast": 0.42193153222019925, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6263200561515986e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014183978782966733, 0.00013419966853689402, 0.00012694251199718565, 0.00012026230979245156, 0.0001100986628443934, 0.00010288084740750492, 9.627256804378703e-05, 8.950204937718809e-05, 7.950894359964877e-05, 7.303059101104736e-05, 6.741164543200284e-05, 6.19373022345826e-05, 5.454086567624472e-05, 4.9690479499986395e-05, 4.586216527968645e-05, 4.195953442831524e-05, 3.827991531579755e-05, 3.556076262611896e-05, 3.321478288853541e-05, 3.127707532257773e-05, 2.9045007977401838e-05, 2.75631136901211e-05, 2.6114066713489592e-05, 2.495846274541691e-05, 2.3635351681150496e-05, 2.2786289264331572e-05, 2.1980353267281316e-05, 2.1307483621058054e-05, 2.0573477740981616e-05, 2.0150520867900923e-05, 1.98783163796179e-05, 1.9670718756970018e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6cc11327437b0ddd855bc16b53a520d7ca542f4fd424dbb567b2c52308727f3b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.005864492617547512, 0.009892112575471401, 0.014509912580251694, 0.019168438389897346, 0.006793031003326178, 0.01223041582852602, 0.017751524224877357, 0.023574935272336006, 0.008267601951956749, 0.014755276031792164, 0.02151178941130638, 0.02830074541270733, 0.009282662533223629, 0.01676790602505207, 0.02516039088368416, 0.0331411249935627, 0.010437680408358574, 0.01991143450140953, 0.029664477333426476, 0.04036801680922508, 0.01357664167881012, 0.025029508396983147, 0.03699437528848648, 0.05095698684453964, 0.015438608825206757, 0.029965175315737724, 0.04443887993693352, 0.060807984322309494, 0.019359532743692398, 0.037317242473363876, 0.05564062297344208, 0.07935244590044022 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3918705690230126e-05, "first_to_last_ratio": 3.3693622509367107, "imbalance_abs_log_ratio": 1.2147234834100593, "last_quartile_mean": 4.1309614857709676e-06, "mean": 7.606254285974501e-06, "normalized": [ 2.2824394746096446, 2.1562501675642154, 2.0296925788947253, 1.9095870929636571, 1.7365187636180548, 1.6206280695592366, 1.5089330358597304, 1.395172874910497, 1.24547847149719, 1.1599616848108578, 1.0770525233380077, 0.9985747583354554, 0.9126325944161369, 0.8598702731715868, 0.8109135685663204, 0.7673206825431794, 0.7307283758200082, 0.6989328774642551, 0.673329950916721, 0.6494877214192842, 0.6312478565559216, 0.6141082928304196, 0.5996495921413664, 0.5866836875352973, 0.57380070608062, 0.5636230401273656, 0.554819016803077, 0.5473940766317459, 0.534486881856006, 0.5284700210369294, 0.5232880116136952, 0.5189232765087916 ], "peak_layer": 1, "peak_normalized": 2.2824394746096446, "population_cv": 0.5333416755025421, "rest_mean": 8.168133886270138e-06, "spike_contrast": 0.5597489595589675, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.572104444378056e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.7360815036226995e-05, 1.640098707866855e-05, 1.5438357877428643e-05, 1.4524805010296404e-05, 1.320840328844497e-05, 1.2326909200055525e-05, 1.147732837125659e-05, 1.0612039659463335e-05, 9.473425961914472e-06, 8.82296353665879e-06, 8.192335371859372e-06, 7.595413535455009e-06, 6.9417155827977695e-06, 6.540391950693447e-06, 6.168014806462452e-06, 5.836436230310937e-06, 5.558105840464123e-06, 5.3162611948209815e-06, 5.121518825035309e-06, 4.940168764733244e-06, 4.801431714440696e-06, 4.671063834393863e-06, 4.561087280308129e-06, 4.46246531282668e-06, 4.36447407992091e-06, 4.287060164642753e-06, 4.220094524498563e-06, 4.1636185414972715e-06, 4.065443135914393e-06, 4.019677362521179e-06, 3.980261681135744e-06, 3.947062396036927e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.641166717396118e-05, "first_to_last_ratio": 3.40273807382454, "imbalance_abs_log_ratio": 1.2245804233600555, "last_quartile_mean": 1.6578316035520402e-05, "mean": 3.084058886315688e-05, "normalized": [ 2.254829765809383, 2.1412750915381364, 2.0245423189025695, 1.913826251974467, 1.7440086673720954, 1.6283807150646488, 1.518515381022133, 1.407719100788124, 1.2571699318719751, 1.1712526341901288, 1.0875882687575642, 1.0075046017085245, 0.9193296533476374, 0.8658469008110146, 0.8153151132261776, 0.770486727612793, 0.7322862023214354, 0.6994539525278819, 0.6731242297658762, 0.648415700488765, 0.6288998621420432, 0.6111632256967778, 0.5959791417905044, 0.5826975877435189, 0.5692453444999938, 0.5587842318415797, 0.5495883661230995, 0.5418092679201958, 0.5286894380532074, 0.5225725836741036, 0.517092481391006, 0.5126072600226392 ], "peak_layer": 1, "peak_normalized": 2.254829765809383, "population_cv": 0.5330470009230729, "rest_mean": 3.313880080424025e-05, "spike_contrast": 0.5561530270868602, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8430244381306693e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.95402777637355e-05, 6.603818474104628e-05, 6.243807729333639e-05, 5.902352859266102e-05, 5.3786254284204915e-05, 5.022022014600225e-05, 4.6831908548483625e-05, 4.3414886022219434e-05, 3.877186099998653e-05, 3.6122120945947245e-05, 3.3541862649144605e-05, 3.1072035199031234e-05, 2.835266786860302e-05, 2.6703228286351077e-05, 2.5144798200926743e-05, 2.3762264390825294e-05, 2.2584137695957907e-05, 2.1571571778622456e-05, 2.0759547624038532e-05, 1.9997522031189874e-05, 1.9395642084418796e-05, 1.8848633771995082e-05, 1.8380347682978027e-05, 1.797073673515115e-05, 1.755586163199041e-05, 1.7233234757441096e-05, 1.6949628843576647e-05, 1.6709716874174774e-05, 1.630509359529242e-05, 1.611644620425068e-05, 1.594743662280962e-05, 1.580910975462757e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05002643074840307, "first_to_last_ratio": 3.402738130192313, "imbalance_abs_log_ratio": 1.2245804399254718, "last_quartile_mean": 0.014701816253364086, "mean": 0.027349742653314024, "normalized": [ 2.254829786736978, 2.1412751957662834, 2.02454215992989, 1.9138263077398268, 1.7440085434241568, 1.628380710143593, 1.5185154852595628, 1.4077191333516668, 1.2571699503960287, 1.171252642415901, 1.0875882368603997, 1.0075045704457566, 0.9193296608188956, 0.8658469286585574, 0.8153151036674042, 0.7704867722207724, 0.7322861827710381, 0.6994539437935281, 0.673124202328064, 0.6484156924895115, 0.6288998911838509, 0.6111632562306977, 0.5959791161376584, 0.5826976161606044, 0.5692453437279956, 0.5587841942711541, 0.5495883614451207, 0.5418092759796765, 0.5286894564849702, 0.5225725676043702, 0.5170924573716015, 0.5126072541844844 ], "peak_layer": 1, "peak_normalized": 2.254829786736978, "population_cv": 0.5330470020894653, "rest_mean": 0.029387819925668062, "spike_contrast": 0.5561530397267354, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.016344125382602215, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.061669014394283295, 0.05856332555413246, 0.05537070706486702, 0.05234265699982643, 0.047698184847831726, 0.044535793364048004, 0.04153100773692131, 0.03850075602531433, 0.034383274614810944, 0.03203345835208893, 0.029745258390903473, 0.027554990723729134, 0.02514342963695526, 0.023680690675973892, 0.02229865826666355, 0.021072614938020706, 0.02002783864736557, 0.019129885360598564, 0.01840977370738983, 0.017734002321958542, 0.017200250178575516, 0.016715157777071, 0.01629987545311451, 0.015936629846692085, 0.015568713657557964, 0.015282603912055492, 0.015031100250780582, 0.01481834426522255, 0.0144595205783844, 0.014292225241661072, 0.014142345637083054, 0.014019676484167576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.375096921194199e-06, "first_to_last_ratio": 3.8221077500661207, "imbalance_abs_log_ratio": 1.3408020374840417, "last_quartile_mean": 1.1446817325122538e-06, "mean": 2.265028140158165e-06, "normalized": [ 2.626026515434601, 2.4001979720003934, 2.1920710272657535, 1.9791200058879264, 1.7560134880541989, 1.6114191789724852, 1.5125696765745171, 1.375271110839599, 1.211780627728412, 1.1344335816532227, 1.0401368166449354, 0.9675242038709502, 0.8757726873912133, 0.8237780432734694, 0.7832771211345639, 0.7406910125458439, 0.7019739226432762, 0.6704065668591555, 0.6413840580656591, 0.6173312336057731, 0.5969185512142089, 0.5799188408093214, 0.5668593216879733, 0.5521486786148365, 0.5377886785355955, 0.5258358481305954, 0.517639304639018, 0.5093446353167698, 0.49603515883647503, 0.4905101977434218, 0.48510625014282244, 0.48071568388301256 ], "peak_layer": 1, "peak_normalized": 2.626026515434601, "population_cv": 0.6051676351147116, "rest_mean": 2.446764428727706e-06, "spike_contrast": 0.5246325174631261, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.283652181882644e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.9480239542608615e-06, 5.436515948531451e-06, 4.9651025619823486e-06, 4.4827625060861465e-06, 3.977419964940054e-06, 3.6499097859632457e-06, 3.426012881391216e-06, 3.1150277663982706e-06, 2.744717221503379e-06, 2.569523985584965e-06, 2.3559391593153123e-06, 2.1914695480518276e-06, 1.983649781323038e-06, 1.8658804492588388e-06, 1.7741447209118633e-06, 1.677685986578581e-06, 1.5899906884442316e-06, 1.5184897392828134e-06, 1.4527529401675565e-06, 1.39827261591563e-06, 1.352037315882626e-06, 1.3135324934410164e-06, 1.2839523151342291e-06, 1.2506322946137516e-06, 1.2181064903415972e-06, 1.1910329931197339e-06, 1.1724675914592808e-06, 1.153679932031082e-06, 1.123533593272441e-06, 1.1110194009233965e-06, 1.0987793075400987e-06, 1.0888345514104003e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.626059489964973e-05, "first_to_last_ratio": 2.8653088341201625, "imbalance_abs_log_ratio": 1.0526761398616047, "last_quartile_mean": 1.614506413716299e-05, "mean": 2.7227290900100343e-05, "normalized": [ 2.069219136955177, 1.9687798201242246, 1.8639507108542548, 1.7654020485254056, 1.6236839316541174, 1.525302822734854, 1.4342547809805677, 1.3418262274018482, 1.2243702183610123, 1.1518310829140999, 1.0836883538565836, 1.0178339180148281, 0.949221197634969, 0.9036382524726367, 0.8600878611114346, 0.8201852972104389, 0.7888274227376761, 0.7582177263486256, 0.7331791140207194, 0.7093278728046818, 0.6906726578980285, 0.6727165175040963, 0.657004042862766, 0.6429903513549862, 0.6287412299367678, 0.6172859346889895, 0.6070185450137164, 0.5981756933578195, 0.58312260952694, 0.5759704572874644, 0.569432199896746, 0.5640419639535237 ], "peak_layer": 1, "peak_normalized": 2.069219136955177, "population_cv": 0.45293224377011354, "rest_mean": 2.89495433039351e-05, "spike_contrast": 0.6192542566623438, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.792712791939266e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.6339231377933174e-05, 5.360454088076949e-05, 5.075032822787762e-05, 4.806711513083428e-05, 4.42085147369653e-05, 4.1529863665346056e-05, 3.905087214661762e-05, 3.6534293030854315e-05, 3.333628410473466e-05, 3.1361239962279797e-05, 2.9505898055504076e-05, 2.771286017377861e-05, 2.5844721676548943e-05, 2.4603621568530798e-05, 2.3417862394126132e-05, 2.2331423679133877e-05, 2.1477633708855137e-05, 2.0644214600906707e-05, 1.9962481019319966e-05, 1.9313076336402446e-05, 1.8805145373335108e-05, 1.8316248315386474e-05, 1.7888440197566524e-05, 1.7506885342299938e-05, 1.7118920368375257e-05, 1.680702371231746e-05, 1.652747050684411e-05, 1.628670361242257e-05, 1.5876848920015618e-05, 1.5682115190429613e-05, 1.550409615447279e-05, 1.53573346324265e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.757111198865459e-05, "first_to_last_ratio": 2.261484300797815, "imbalance_abs_log_ratio": 0.8160213679695097, "last_quartile_mean": 1.6613474599580513e-05, "mean": 2.4096830372855038e-05, "normalized": [ 1.8905505844846502, 1.7997195920380655, 1.703326591013662, 1.610437077165417, 1.490662081122157, 1.4031283462782311, 1.3280828236310434, 1.2474718200193673, 1.158269301434096, 1.0891577101716936, 1.0361895464823103, 0.9872929826228773, 0.9419073791196091, 0.9071102148715466, 0.8752487270925183, 0.8477505951103422, 0.8347755693998784, 0.8134040107759765, 0.7935380337199553, 0.7733807536170327, 0.7587708376785838, 0.7447349978405977, 0.7306079701092049, 0.7189107072064382, 0.711201857826888, 0.7050007791285537, 0.7000690842179099, 0.6958731578478594, 0.6835994787579611, 0.6787755788074042, 0.6729650681227183, 0.6680867422854508 ], "peak_layer": 1, "peak_normalized": 1.8905505844846502, "population_cv": 0.3606253067246935, "rest_mean": 2.5288975208510505e-05, "spike_contrast": 0.6982982946012236, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.765924826031551e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.5556276745628566e-05, 4.3367537728045136e-05, 4.104477193322964e-05, 3.8806429074611515e-05, 3.592023131204769e-05, 3.381094575161114e-05, 3.200258652213961e-05, 3.0060116841923445e-05, 2.791061888274271e-05, 2.6245248591294512e-05, 2.4968883735709824e-05, 2.379063153057359e-05, 2.269698234158568e-05, 2.1858480977243744e-05, 2.1090720110805705e-05, 2.0428102288860828e-05, 2.0115445295232348e-05, 1.9600458472268656e-05, 1.9121751392958686e-05, 1.8636024833540432e-05, 1.8283972167409956e-05, 1.7945752915693447e-05, 1.7605336324777454e-05, 1.7323469364782795e-05, 1.7137710528913885e-05, 1.69882841873914e-05, 1.6869445971678942e-05, 1.6768337445682846e-05, 1.647258068260271e-05, 1.6356339983758517e-05, 1.6216325093409978e-05, 1.6098772903205827e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001084787909348961, "first_to_last_ratio": 5.225248440781252, "imbalance_abs_log_ratio": 1.6535023451953625, "last_quartile_mean": 2.0760503957717447e-05, "mean": 5.3153509043113445e-05, "normalized": [ 2.5237742282686555, 2.39597413894643, 2.265748981789298, 2.151554924730914, 1.9389326261416564, 1.811160187739653, 1.6845666093621592, 1.5551563330371112, 1.3755697660543504, 1.2684162903360312, 1.1635545557382814, 1.0626373891730778, 0.9348691942291281, 0.8623306324991865, 0.788381531623569, 0.7297638393679164, 0.6688462612899458, 0.6250625348617511, 0.5885533959524297, 0.5510274355176712, 0.5186065946717524, 0.49092990570860817, 0.46938426076169515, 0.4505875210977826, 0.4285538100318043, 0.4136111569238702, 0.40105822090753923, 0.39087833614351924, 0.37910335602991646, 0.3738905758036781, 0.37037507613467424, 0.36714032912594324 ], "peak_layer": 1, "peak_normalized": 2.5237742282686555, "population_cv": 0.6747695646712838, "rest_mean": 5.835455609754987e-05, "spike_contrast": 0.42957836757855616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5067854949156754e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013414745626505464, 0.00012735443306155503, 0.00012043250899296254, 0.00011436269414843991, 0.00010306107287760824, 9.626951941754669e-05, 8.954062650445849e-05, 8.266201621154323e-05, 7.311635999940336e-05, 6.742077675880864e-05, 6.184700760059059e-05, 5.648290607496165e-05, 4.969157816958614e-05, 4.583589907269925e-05, 4.190524487057701e-05, 3.878950883517973e-05, 3.555152579792775e-05, 3.32242670992855e-05, 3.12836782541126e-05, 2.9289041776792146e-05, 2.756576031970326e-05, 2.6094647182617337e-05, 2.4949420549091883e-05, 2.3950307877385058e-05, 2.2779138816986233e-05, 2.198488436988555e-05, 2.1317651771823876e-05, 2.0776555174961686e-05, 2.015067366301082e-05, 1.9873596102115698e-05, 1.968673495866824e-05, 1.951479680428747e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a1eaf8973066bfbad01efbb61e56182f134640c86d8dd38e1601245f8a0a6387", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02856997773051262, 0.015031684190034866, 0.015670204535126686, 0.016530683264136314, 0.01773575134575367, 0.01292925700545311, 0.014021075330674648, 0.015247474424540997, 0.01669454388320446, 0.01373021025210619, 0.01499613095074892, 0.01623532734811306, 0.01758282259106636, 0.015234442427754402, 0.01637510396540165, 0.01772587187588215, 0.01901337131857872, 0.017032917588949203, 0.01834850199520588, 0.019697511568665504, 0.021141810342669487, 0.019469235092401505, 0.02084389142692089, 0.022254284471273422, 0.02407064661383629, 0.022380243986845016, 0.023947052657604218, 0.025450291112065315, 0.027382872998714447, 0.025761734694242477, 0.02756396308541298, 0.02946169301867485 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.9468072852978366e-06, "first_to_last_ratio": 6.831203575113915, "imbalance_abs_log_ratio": 1.9215008769619122, "last_quartile_mean": 4.313745378681233e-07, "mean": 1.4344362195828353e-06, "normalized": [ 2.0208098689060576, 2.70247744984379, 2.3511261487968675, 2.0828624204646062, 1.7252416990779311, 2.0561625392735503, 1.8889131659507596, 1.6070574509858848, 1.2980615117448049, 1.4730921817018607, 1.3154183683511145, 1.1251444251921345, 0.9479215504591184, 0.9438177843313365, 0.9908118070600781, 0.7983125303617123, 0.663485001791352, 0.7166985535761707, 0.6372153189923393, 0.4803275608465089, 0.44713051888100325, 0.4971914883998871, 0.44243986640593336, 0.382460121340808, 0.33911149104781046, 0.37657627859451315, 0.3537072871377602, 0.301474679623081, 0.272351762260187, 0.2855781634777493, 0.2572154244268084, 0.21980558069648082 ], "peak_layer": 2, "peak_normalized": 2.70247744984379, "population_cv": 0.7068955562190269, "rest_mean": 1.588062559676473e-06, "spike_contrast": 0.3808754128681272, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.048539830771916e-07, "top_five_layers": [ 2, 3, 4, 6, 1 ], "top_five_spike_overlap": 0 }, "values": [ 2.8987228688492905e-06, 3.876531536661787e-06, 3.3725405046425294e-06, 2.9877332963224035e-06, 2.4747491806920152e-06, 2.9494340196833946e-06, 2.7095254608866526e-06, 2.3052214146446204e-06, 1.861986447693198e-06, 2.1130567802174482e-06, 1.8868837514673942e-06, 1.6139479157573078e-06, 1.3597330053016776e-06, 1.35384641453129e-06, 1.421256342837296e-06, 1.145128408097662e-06, 9.517269177194976e-07, 1.0280583637722884e-06, 9.140447332356416e-07, 6.889992505421105e-07, 6.413802111637779e-07, 7.131894790290971e-07, 6.346517693600617e-07, 5.486146505973011e-07, 4.864338052357198e-07, 5.40174653451686e-07, 5.073705438007892e-07, 4.3244619973847875e-07, 3.9067123225322575e-07, 4.0964366121443163e-07, 3.689591210331855e-07, 3.1529708621746977e-07 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.710511570418021e-06, "first_to_last_ratio": 6.354606328287203, "imbalance_abs_log_ratio": 1.8491799559542976, "last_quartile_mean": 1.0560074414911469e-06, "mean": 3.2492914723292188e-06, "normalized": [ 1.783276830307924, 2.779944343220534, 2.3950635818648984, 2.1106770285738783, 1.7960465411480506, 2.159230004292565, 1.9196468120338586, 1.5778990753682762, 1.2819032996284179, 1.4557156863533565, 1.3120099400931993, 1.1009867702080869, 0.8992855746074583, 0.9377834334418341, 0.8845056521165512, 0.7577187265725858, 0.6356736108971007, 0.6642988512627761, 0.5934764503981409, 0.5228260514911628, 0.46756909304244726, 0.49649554676318175, 0.45575067618660486, 0.41224667168429796, 0.3602883651000989, 0.3931940610190251, 0.3661495872210781, 0.32797257011813363, 0.29319922194152587, 0.31032847914557965, 0.2896492848736135, 0.25918817902375785 ], "peak_layer": 2, "peak_normalized": 2.779944343220534, "population_cv": 0.7087840069849813, "rest_mean": 3.5871756225518496e-06, "spike_contrast": 0.39716958717329126, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4247170611270122e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 5.794386197521817e-06, 9.032849447976332e-06, 7.782259672239888e-06, 6.858204869786277e-06, 5.835878710058751e-06, 7.015967639745213e-06, 6.237492016225588e-06, 5.1270540097902995e-06, 4.165277459833305e-06, 4.730044565803837e-06, 4.2631027099560015e-06, 3.577426923584426e-06, 2.9220409487606958e-06, 3.047131713174167e-06, 2.8740166726493044e-06, 2.462048996676458e-06, 2.0654888430726714e-06, 2.1585005924862344e-06, 1.9283779693068936e-06, 1.6988142306217924e-06, 1.519268266747531e-06, 1.6132587461470393e-06, 1.4808667856414104e-06, 1.3395095947998925e-06, 1.1706819122991874e-06, 1.2776021094396128e-06, 1.1897267313543125e-06, 1.0656784752427484e-06, 9.526897315481619e-07, 1.0083476809086278e-06, 9.411549513060891e-07, 8.421779398304352e-07 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.005950948863755912, "first_to_last_ratio": 6.354606465537837, "imbalance_abs_log_ratio": 1.8491799775529028, "last_quartile_mean": 0.0009364779543830082, "mean": 0.0028815042023779824, "normalized": [ 1.7832768321055497, 2.779944541805248, 2.3950634036601817, 2.110677044556637, 1.796046582443378, 2.1592300209325574, 1.9196467896009393, 1.5778990914249873, 1.281903295419947, 1.4557156625259682, 1.3120099546166246, 1.1009867557766109, 0.8992854956916723, 0.9377834188412186, 0.884505639688408, 0.7577187722730399, 0.635673635605656, 0.6642988751267256, 0.5934764053600845, 0.5228260433663814, 0.46756910562655507, 0.49649557415115175, 0.45575067494350857, 0.4122466780510639, 0.36028835142322546, 0.3931940647806284, 0.3661495503121345, 0.32797257179832034, 0.2931992062983405, 0.3103284950057511, 0.2896492814770661, 0.25918818531043897 ], "peak_layer": 2, "peak_normalized": 2.779944541805248, "population_cv": 0.7087840141850674, "rest_mean": 0.0031811432455907816, "spike_contrast": 0.3971695932838214, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0012634533690288664, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.005138519685715437, 0.008010421879589558, 0.006901385262608528, 0.006081924773752689, 0.005175315774977207, 0.006221830379217863, 0.005531470291316509, 0.004546722862869501, 0.0036938097327947617, 0.004194650799036026, 0.00378056219778955, 0.0031724979635328054, 0.0025912949349731207, 0.0027022268623113632, 0.002548706717789173, 0.0021833698265254498, 0.0018316962523385882, 0.0019141800003126264, 0.0017101047560572624, 0.0015065254410728812, 0.0013473023427650332, 0.001430654083378613, 0.0013132474850863218, 0.0011878905352205038, 0.0010381723986938596, 0.0011329903500154614, 0.001055061467923224, 0.0009450543439015746, 0.0008448547450825572, 0.0008942128624767065, 0.000834625621791929, 0.0007468518451787531 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7645694729017123e-06, "first_to_last_ratio": 5.854487285906525, "imbalance_abs_log_ratio": 1.7672084246254944, "last_quartile_mean": 4.722137632029444e-07, "mean": 1.404573582775015e-06, "normalized": [ 1.7305940535699378, 2.5618749599510693, 2.200447296481188, 1.980294414184873, 1.7177222534372074, 2.006950537243153, 1.8986342274283854, 1.6495820380503576, 1.3071680990137817, 1.5435943853891647, 1.37926355112175, 1.1804343531650965, 0.9505292781052922, 0.987344076938019, 0.9642189006877869, 0.8086506810691724, 0.6736023260407531, 0.710546872437511, 0.640481569051335, 0.5322781307138237, 0.48482706689793736, 0.5138657129886831, 0.47272339176998346, 0.4147938968760285, 0.36560568881857053, 0.4082117778611776, 0.3821008292991752, 0.34425699387548675, 0.3079872683634573, 0.32257229483060335, 0.30100273359597224, 0.25784034074326717 ], "peak_layer": 2, "peak_normalized": 2.5618749599510693, "population_cv": 0.6600852637188594, "rest_mean": 1.5475375452869802e-06, "spike_contrast": 0.40875789226367176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.325681852104025e-07, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 2.4307466901518637e-06, 3.5983418911200715e-06, 3.090690142926178e-06, 2.7814692202809965e-06, 2.4126672997226706e-06, 2.8189097065478563e-06, 2.66677147919836e-06, 2.316959353265702e-06, 1.836013780120993e-06, 2.1680918962374562e-06, 1.9372771475900663e-06, 1.658006908655807e-06, 1.3350883136808989e-06, 1.3867974075765233e-06, 1.3543163959184312e-06, 1.1358093843227834e-06, 9.461240324526443e-07, 9.980153663491365e-07, 8.996034921437968e-07, 7.476238010895031e-07, 6.809752903791377e-07, 7.217622055577522e-07, 6.639747880399227e-07, 5.826085498483735e-07, 5.135200922268268e-07, 5.733634793614328e-07, 5.36688730790047e-07, 4.835342792830488e-07, 4.325907809743512e-07, 4.5307652385417896e-07, 4.227804879519681e-07, 3.621557311817014e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.024905337018936e-06, "first_to_last_ratio": 5.991275482114614, "imbalance_abs_log_ratio": 1.7903043247043764, "last_quartile_mean": 6.717944032175183e-07, "mean": 2.0148276771436713e-06, "normalized": [ 1.5675942414338733, 2.5250440978186566, 2.231455540086097, 2.0780670863790145, 1.7996941190454676, 2.2162616169225418, 1.9414222145713715, 1.621600841461054, 1.3376231697058878, 1.5231899507700504, 1.3715764047674524, 1.1419542658104653, 0.9487730313859013, 0.9689325883752917, 0.9148133638638105, 0.7812038234732276, 0.6554220232920718, 0.683968041755189, 0.6172110636131051, 0.5352507957421382, 0.47670028302923295, 0.5067173036495194, 0.46642566869070556, 0.4216965387703109, 0.36513142756884537, 0.4048860931364547, 0.37578964789761116, 0.34071121185725145, 0.3026913075409305, 0.3181401874842009, 0.29774178307895716, 0.26231026702331217 ], "peak_layer": 2, "peak_normalized": 2.5250440978186566, "population_cv": 0.6756995388559863, "rest_mean": 2.2210362439417857e-06, "spike_contrast": 0.40580221006851624, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.013014164338528e-07, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 3.1584322641720064e-06, 5.087528734293301e-06, 4.495998382481048e-06, 4.186947080597747e-06, 3.6260735214455053e-06, 4.465385245566722e-06, 3.911631210939959e-06, 3.2672462566551985e-06, 2.6950801839120686e-06, 3.0689652703586034e-06, 2.763490101642674e-06, 2.3008410607872065e-06, 1.911614162963815e-06, 1.952232196344994e-06, 1.8431912849337095e-06, 1.5739910850243177e-06, 1.3205624327383703e-06, 1.378077740810113e-06, 1.2435739336069673e-06, 1.078438117474434e-06, 9.6046892394952e-07, 1.0209480478806654e-06, 9.397673466082779e-07, 8.496458576701116e-07, 7.356769060606894e-07, 8.157757065418991e-07, 7.57151383368182e-07, 6.864743795631512e-07, 6.098708240642736e-07, 6.409976549548446e-07, 5.9989838518959e-07, 5.285099859975162e-07 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.5129092762199434e-06, "first_to_last_ratio": 5.690023270721292, "imbalance_abs_log_ratio": 1.7387143378875727, "last_quartile_mean": 4.4163427048715675e-07, "mean": 1.3025998217841561e-06, "normalized": [ 1.3333242329011745, 2.2929798702209006, 2.055055088077371, 2.0612267847935852, 1.8004956832661445, 2.300472589669482, 1.9555756876630987, 1.6340628779322937, 1.3881426615583095, 1.6201188266700812, 1.4367832841182007, 1.1670671159103638, 1.0033834447630112, 1.0052901825823841, 0.9417324589159911, 0.8084681814074165, 0.6736539539316041, 0.710866891998549, 0.6407147024486859, 0.5472927533253719, 0.486548923284392, 0.5119738979313811, 0.4796067027050235, 0.43283812540794353, 0.36960358799802223, 0.40990700009728237, 0.38098457936452723, 0.3475616176385816, 0.31215171064803027, 0.32339061526715857, 0.3036943199348575, 0.2650316475687802 ], "peak_layer": 6, "peak_normalized": 2.300472589669482, "population_cv": 0.6517350453796433, "rest_mean": 1.4337971336957438e-06, "spike_contrast": 0.41437824326663764, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.941343374615827e-07, "top_five_layers": [ 6, 2, 4, 3, 7 ], "top_five_spike_overlap": 0 }, "values": [ 1.7367879081575666e-06, 2.9868351703044027e-06, 2.676914391486207e-06, 2.6849536425288534e-06, 2.345325356145622e-06, 2.9965951853228034e-06, 2.547332542235381e-06, 2.128530013578711e-06, 1.8081943835568381e-06, 2.110366494889604e-06, 1.8715536498348229e-06, 1.520221417194989e-06, 1.307007096329471e-06, 1.3094908126731752e-06, 1.226700533152325e-06, 1.0531105090194615e-06, 8.775015203354997e-07, 9.259750868295669e-07, 8.345948572241468e-07, 7.129034429453895e-07, 6.33778540759522e-07, 6.668971082035569e-07, 6.247356054700504e-07, 5.638148650177754e-07, 4.814455678570084e-07, 5.33944785274798e-07, 4.962704451827449e-07, 4.527337011950294e-07, 4.0660876265974366e-07, 4.212485578136693e-07, 3.955921670240059e-07, 3.4523017689025437e-07 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 1.2106409826628806e-05, "first_to_last_ratio": 6.182837050272253, "imbalance_abs_log_ratio": 1.8217772357527195, "last_quartile_mean": 1.9580671022367824e-06, "mean": 5.891290232540314e-06, "normalized": [ 1.6628561901458256, 2.679541050672962, 2.4593584605705066, 2.112226792311699, 1.796452079584457, 2.148898726796378, 1.979566597291414, 1.6008399202436439, 1.3322729283852512, 1.4727285184978303, 1.3085955832877125, 1.1187775970056968, 0.8873455238855094, 0.9252291834138445, 0.8986949676582121, 0.75315156809204, 0.6225658481990487, 0.659474986853632, 0.5815564660361224, 0.5264648905960624, 0.46580075393312903, 0.4965536609025122, 0.450087000441461, 0.40202926580138304, 0.37131699924705175, 0.4000060060879296, 0.37557443972643606, 0.33475575489567644, 0.30250455585342495, 0.3159959000194193, 0.2955172411886233, 0.2632605423751041 ], "peak_layer": 2, "peak_normalized": 2.679541050672962, "population_cv": 0.7042782314383109, "rest_mean": 6.505339919621528e-06, "spike_contrast": 0.3958935204190824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5754219223017572e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 9.796368431125302e-06, 1.5785954019520432e-05, 1.4488794477074407e-05, 1.2443741070455872e-05, 1.0583420589682646e-05, 1.265978607989382e-05, 1.1662201359285973e-05, 9.431012585991994e-06, 7.848806490073912e-06, 8.676271136209834e-06, 7.709316378168296e-06, 6.591043529624585e-06, 5.227610017755069e-06, 5.450793651107233e-06, 5.294472884997958e-06, 4.437034476723056e-06, 3.6677161006082315e-06, 3.885158548655454e-06, 3.426117928029271e-06, 3.1015574677439872e-06, 2.744167431956157e-06, 2.925341732407105e-06, 2.651593149494147e-06, 2.3684710868110415e-06, 2.187536210840335e-06, 2.356551476623281e-06, 2.2126180283521535e-06, 1.972143309103558e-06, 1.782142135198228e-06, 1.8616235593071906e-06, 1.7409778365617967e-06, 1.5509442619077163e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "51c96475782d47da4b4460ab98a6f5efa352f41865c65bc22f8b46e27bda20aa", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.013703026808798313, 0.014947476796805859, 0.015775443986058235, 0.016744175925850868, 0.012103146873414516, 0.013081531040370464, 0.01416850183159113, 0.015494133345782757, 0.012773395515978336, 0.013903867453336716, 0.015176740474998951, 0.016410861164331436, 0.014334342442452908, 0.015468464232981205, 0.01670125499367714, 0.017849057912826538, 0.01606663130223751, 0.0173187293112278, 0.018571525812149048, 0.019770435988903046, 0.018272221088409424, 0.01950262486934662, 0.02077464945614338, 0.0220524650067091, 0.020764723420143127, 0.02203267626464367, 0.023422541096806526, 0.024811791256070137, 0.023658527061343193, 0.025185640901327133, 0.026624467223882675, 0.02854793891310692 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.74612942677777e-06, "first_to_last_ratio": 4.290754879791884, "imbalance_abs_log_ratio": 1.4564626801157643, "last_quartile_mean": 1.8053069084089657e-06, "mean": 4.49623530585086e-06, "normalized": [ 1.8981071323025052, 1.739149695276559, 1.5735966428132337, 1.5026215379163184, 1.9831005875602767, 1.854738075795215, 1.6911251838970416, 1.5399882637631246, 1.7056874629197014, 1.4591941238713075, 1.3137324489958155, 1.17819115097449, 1.2205893686523535, 1.0594273921988615, 0.9634337939297237, 0.867129494870504, 0.8508597774565135, 0.7919049093759021, 0.7029364663408488, 0.6435982792516399, 0.6460999667655339, 0.576190114496073, 0.5231361846468904, 0.5033401852027535, 0.47554278072756984, 0.4897885434342915, 0.44031530870935176, 0.391536447771622, 0.3980832380767055, 0.38067347531577295, 0.33586151098891653, 0.3003204557025833 ], "peak_layer": 5, "peak_normalized": 1.9831005875602767, "population_cv": 0.5381157672273565, "rest_mean": 4.875199793888088e-06, "spike_contrast": 0.5025080353673121, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4498270704498283e-06, "top_five_layers": [ 5, 1, 6, 2, 9 ], "top_five_spike_overlap": 0 }, "values": [ 8.534336302545853e-06, 7.81962626206223e-06, 7.0752607825852465e-06, 6.756140010111267e-06, 8.9164868768421e-06, 8.339338819496334e-06, 7.603696758451406e-06, 6.924149602127727e-06, 7.66917219152674e-06, 6.560880137840286e-06, 5.906850219616899e-06, 5.297424650052562e-06, 5.4880570132809225e-06, 4.7634348447900265e-06, 4.331825039116666e-06, 3.898818249581382e-06, 3.8256657717283815e-06, 3.5605908124125563e-06, 3.1605677577317692e-06, 2.8937693059560843e-06, 2.905017481680261e-06, 2.590686335679493e-06, 2.352143383177463e-06, 2.2631359115621308e-06, 2.1381522401497932e-06, 2.202204541390529e-06, 1.979761236725608e-06, 1.7604399999981979e-06, 1.7898759097079164e-06, 1.711597519715724e-06, 1.510112383584783e-06, 1.3503114359991741e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.130438562948257e-05, "first_to_last_ratio": 4.214157668538487, "imbalance_abs_log_ratio": 1.4384497300922356, "last_quartile_mean": 7.428385004004667e-06, "mean": 1.8293917648293245e-05, "normalized": [ 1.8783239189466776, 1.7239775883511843, 1.5628409896597186, 1.5021774100270706, 1.9687179771495062, 1.8492004689267585, 1.6770781946618867, 1.5272107503967014, 1.6975023935986964, 1.4550858154880342, 1.3108870533979449, 1.1834221941571295, 1.2280669813964384, 1.0632172174009653, 0.9681770292934567, 0.8701237722001045, 0.8538823529559815, 0.7967879122792568, 0.7086904702827618, 0.651045975234203, 0.652290955492651, 0.5835747270147889, 0.5308413464878547, 0.5084153950901334, 0.48184390105002417, 0.49559938674797777, 0.443562473293985, 0.39641716095408386, 0.40208621599683964, 0.38482821036485854, 0.3408788461690432, 0.30324491553328303 ], "peak_layer": 5, "peak_normalized": 1.9687179771495062, "population_cv": 0.5313999208641894, "rest_mean": 1.9813690734601415e-05, "spike_contrast": 0.5090996481848562, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0087142982229125e-05, "top_five_layers": [ 5, 1, 6, 2, 9 ], "top_five_spike_overlap": 0 }, "values": [ 3.4361903090029955e-05, 3.153830402879976e-05, 2.859048436221201e-05, 2.7480709832161665e-05, 3.601556454668753e-05, 3.3829121093731374e-05, 3.068033038289286e-05, 2.7938667699345388e-05, 3.1053968996275216e-05, 2.6619220079737715e-05, 2.3981259801075794e-05, 2.1649428163073026e-05, 2.246615622425452e-05, 1.9450408217380755e-05, 1.7711750842863694e-05, 1.5917972632450983e-05, 1.5620853446307592e-05, 1.4576372450392228e-05, 1.2964725101483054e-05, 1.1910181456187274e-05, 1.1932957022509072e-05, 1.0675867997633759e-05, 9.711167876957916e-06, 9.300909368903376e-06, 8.814812645141501e-06, 9.06645436771214e-06, 8.114495358313434e-06, 7.252022896864219e-06, 7.355732122960035e-06, 7.040015589154791e-06, 6.236009539861698e-06, 5.547537512029521e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027761041885241866, "first_to_last_ratio": 4.214157540695516, "imbalance_abs_log_ratio": 1.438449699755693, "last_quartile_mean": 0.006587566225789487, "mean": 0.016223229031311348, "normalized": [ 1.8783239576926372, 1.7239774602868518, 1.5628410093355816, 1.502177362926827, 1.968717976965925, 1.8492004888075664, 1.6770782262083783, 1.5272107169113356, 1.697502508427555, 1.455085816293378, 1.3108869922362147, 1.183422230798549, 1.2280668728519175, 1.0632172396078363, 0.9681770638198094, 0.8701237808458149, 0.8538823828074513, 0.7967879426055737, 0.7086904796790885, 0.6510459681632486, 0.6522909479780994, 0.5835746869352977, 0.5308413214129329, 0.5084153812334933, 0.4818439286563117, 0.4955993855362714, 0.44356247849658154, 0.3964171704578387, 0.40208622884148165, 0.38482821497145886, 0.3408788545627423, 0.30324492364595074 ], "peak_layer": 5, "peak_normalized": 1.968717976965925, "population_cv": 0.5313999193824042, "rest_mean": 0.017570979030871833, "spike_contrast": 0.5090996362791106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00894537903368473, "top_five_layers": [ 5, 1, 6, 2, 9 ], "top_five_spike_overlap": 0 }, "values": [ 0.03047247976064682, 0.027968481183052063, 0.025354327633976936, 0.024370167404413223, 0.03193896263837814, 0.030000003054738045, 0.027207624167203903, 0.024776289239525795, 0.027538971975445747, 0.0236061904579401, 0.021266819909214973, 0.019198929890990257, 0.01992321014404297, 0.017248816788196564, 0.01570695824921131, 0.014116217382252216, 0.013852729462087154, 0.012926473282277584, 0.011497247964143753, 0.010562067851424217, 0.010582265444099903, 0.009467465803027153, 0.008611960336565971, 0.008248139172792435, 0.00781706441193819, 0.008040222339332104, 0.007196015678346157, 0.006431166548281908, 0.006523136980831623, 0.006243156269192696, 0.0055301557295024395, 0.004919611848890781 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.045660991711884e-06, "first_to_last_ratio": 3.917022593501096, "imbalance_abs_log_ratio": 1.3653318227296314, "last_quartile_mean": 5.222489640743788e-07, "mean": 1.2342234425588572e-06, "normalized": [ 1.8213398189861276, 1.6645319684866091, 1.4617715403691671, 1.4547475368447418, 1.9368035492846856, 1.8317059236170603, 1.6013392411126162, 1.4873432240414948, 1.6809680722553373, 1.3943888205708208, 1.3087026096396304, 1.1611861005014494, 1.226147655357962, 1.097886062602167, 1.0319446198761002, 0.9091144134925789, 0.8857120604478002, 0.8417980143622228, 0.7482542320108061, 0.6799549053136131, 0.6827139372140408, 0.6176009501656174, 0.5476403828473109, 0.5412865865177412, 0.5057577756670864, 0.5227785606025747, 0.47425131333003206, 0.40558303186112465, 0.41965194834280006, 0.3928760379302379, 0.35229520413941656, 0.31192390220902655 ], "peak_layer": 5, "peak_normalized": 1.9368035492846856, "population_cv": 0.5025309389629137, "rest_mean": 1.3304471759022785e-06, "spike_contrast": 0.5371241304787221, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.146152825043828e-07, "top_five_layers": [ 5, 6, 1, 9, 2 ], "top_five_spike_overlap": 0 }, "values": [ 2.247940301458584e-06, 2.054404376394814e-06, 1.804152702788997e-06, 1.7954835129785351e-06, 2.390448344158358e-06, 2.2607343908020994e-06, 1.976410430870601e-06, 1.8357138742430834e-06, 2.074690200970508e-06, 1.7209873703905032e-06, 1.6152314401551848e-06, 1.433163106412394e-06, 1.513340180281375e-06, 1.3550367157222354e-06, 1.2736502412735717e-06, 1.1220503211006871e-06, 1.0931665883617825e-06, 1.038966843225353e-06, 9.23512914141611e-07, 8.392162840209494e-07, 8.42621545871225e-07, 7.622575708410295e-07, 6.759105986020586e-07, 6.680685942228592e-07, 6.242181029847416e-07, 6.452255547628738e-07, 5.853320885762514e-07, 5.005800858270959e-07, 5.179442723601824e-07, 4.848968160331424e-07, 4.348109996499261e-07, 3.8498379240081704e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4131825966833276e-05, "first_to_last_ratio": 3.655390206056544, "imbalance_abs_log_ratio": 1.2962028470263438, "last_quartile_mean": 6.60170996980014e-06, "mean": 1.4587583734737564e-05, "normalized": [ 1.850614794041918, 1.6756697305772237, 1.532740691958691, 1.4525582288852792, 1.9097065779239775, 1.773563531018016, 1.5841340955031835, 1.4551854143356047, 1.616552010268487, 1.4126736159014799, 1.2481506316585642, 1.139663047863614, 1.206304825795999, 1.058901504186884, 0.9570224096844311, 0.8837934651949423, 0.8892675475561976, 0.8327723366593228, 0.742233508291339, 0.6863736151537033, 0.7069606538289486, 0.6275682542143274, 0.5826463632089418, 0.5544888806084907, 0.5307528105903948, 0.5485698264270499, 0.49089718904506907, 0.4444636283252074, 0.45082728024375657, 0.4306907622876734, 0.381786289322936, 0.34246647943834635 ], "peak_layer": 5, "peak_normalized": 1.9097065779239775, "population_cv": 0.4863857904662779, "rest_mean": 1.5666839654268524e-05, "spike_contrast": 0.5591173435469335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.759601769270375e-06, "top_five_layers": [ 5, 1, 6, 2, 9 ], "top_five_spike_overlap": 0 }, "values": [ 2.699599826883059e-05, 2.4443972506560385e-05, 2.2358983187587e-05, 2.11893147934461e-05, 2.7858004614245147e-05, 2.587200651760213e-05, 2.3108688765205443e-05, 2.122763908118941e-05, 2.358158781134989e-05, 2.0607494661817327e-05, 1.820750185288489e-05, 1.6624930140096694e-05, 1.7597072655917145e-05, 1.5446814359165728e-05, 1.3960644537291955e-05, 1.2892411177745089e-05, 1.2972264812560752e-05, 1.2148136192990933e-05, 1.0827393452927936e-05, 1.0012532584369183e-05, 1.0312847734894603e-05, 9.154704457614571e-06, 8.499402611050755e-06, 8.088652975857258e-06, 7.74240106693469e-06, 8.002308277355041e-06, 7.16100385034224e-06, 6.483650395239238e-06, 6.576480700459797e-06, 6.282737558649387e-06, 5.569339464273071e-06, 4.995758445147658e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8469200995241408e-05, "first_to_last_ratio": 3.207575199084929, "imbalance_abs_log_ratio": 1.1655152619982017, "last_quartile_mean": 5.7579946997066145e-06, "mean": 1.1597108013461366e-05, "normalized": [ 1.81806399507725, 1.625366158769584, 1.5063686258414184, 1.3954385973171766, 1.8443701608420582, 1.6859765108020661, 1.4840450012318553, 1.3809272845448963, 1.548769019794943, 1.3994132960456658, 1.2002346351153552, 1.120141499141413, 1.2021529711241468, 1.0647775737601772, 0.9335838124381102, 0.9011530250739159, 0.9289007857250168, 0.8471652887398277, 0.7610194462198521, 0.7208419881303229, 0.7713258850383539, 0.6593132095004427, 0.6413626788985748, 0.5872674621057953, 0.57359062275249, 0.6016506688370158, 0.5329045048635649, 0.4939332120091125, 0.4968495752468759, 0.4799066396346222, 0.4191450071900165, 0.37404085818808425 ], "peak_layer": 5, "peak_normalized": 1.8443701608420582, "population_cv": 0.44401994718402354, "rest_mean": 1.2356134498582429e-05, "spike_contrast": 0.6068536235719905, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.498364993807627e-06, "top_five_layers": [ 5, 1, 6, 2, 9 ], "top_five_spike_overlap": 0 }, "values": [ 2.108428452629596e-05, 1.8849546904675663e-05, 1.74695196619723e-05, 1.6183052139240317e-05, 2.138935997209046e-05, 1.9552451703930274e-05, 1.721063017612323e-05, 1.601476287760306e-05, 1.796124161046464e-05, 1.6229147149715573e-05, 1.3919250704930164e-05, 1.2990401955903508e-05, 1.3941497854830232e-05, 1.2348340533208102e-05, 1.082687231246382e-05, 1.0450768968439661e-05, 1.0772562745842151e-05, 9.824667358770967e-06, 8.825624718156178e-06, 8.35968239698559e-06, 8.945149602368474e-06, 7.646126505278517e-06, 7.437952262989711e-06, 6.810604190832237e-06, 6.651992407569196e-06, 6.977407792874146e-06, 6.18015110376291e-06, 5.72819681110559e-06, 5.76201819058042e-06, 5.565529136219993e-06, 4.860869921685662e-06, 4.337792233854998e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.355659252221812e-05, "first_to_last_ratio": 4.688745822260651, "imbalance_abs_log_ratio": 1.5451651314001054, "last_quartile_mean": 1.355513711587264e-05, "mean": 3.660362690993679e-05, "normalized": [ 1.8604312730573458, 1.729027777106884, 1.5601949774918833, 1.5188228334740854, 1.995241408516474, 1.8645405891130502, 1.7681697430427887, 1.5943462503396746, 1.722236164777989, 1.4743310838751806, 1.323156933502311, 1.1917678488813146, 1.2713083492940296, 1.0938173223246854, 1.0218191154232952, 0.8916141451419608, 0.8287888488337875, 0.7802753941234802, 0.720014469658079, 0.6521922684818504, 0.6212316142834627, 0.5670057570442619, 0.503851683647319, 0.4832362271528006, 0.44920645672067455, 0.4521315093539983, 0.4042296913438992, 0.3615519057236543, 0.36249030689593087, 0.352153631754142, 0.30928834976812813, 0.2715260698515788 ], "peak_layer": 5, "peak_normalized": 1.995241408516474, "population_cv": 0.5545162509282403, "rest_mean": 3.9824025205032536e-05, "spike_contrast": 0.4824594203498991, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.921347611641977e-05, "top_five_layers": [ 5, 6, 1, 7, 2 ], "top_five_spike_overlap": 0 }, "values": [ 6.809853221056983e-05, 6.328868767013773e-05, 5.710879486287013e-05, 5.559442433877848e-05, 7.30330721125938e-05, 6.824894808232784e-05, 6.472142558777705e-05, 5.835885531269014e-05, 6.304009002633393e-05, 5.396586493588984e-05, 4.843234273721464e-05, 4.362302570370957e-05, 4.653449650504626e-05, 4.003768117399886e-05, 3.740228567039594e-05, 3.2636311516398564e-05, 3.033667780982796e-05, 2.8560909413499758e-05, 2.635514101712033e-05, 2.3872602469054982e-05, 2.2739330233889632e-05, 2.075446718663443e-05, 1.8442799046169966e-05, 1.7688198568066582e-05, 1.644258554733824e-05, 1.6549653082620353e-05, 1.4796272807870992e-05, 1.3234111065685283e-05, 1.3268459952087142e-05, 1.2890100151707884e-05, 1.1321075362502597e-05, 9.938838957168628e-06 ] } } } } } }, "step": 100 }, { "modes": { "learned": { "bits_per_byte": 3.059021014366605, "forward": { "activation_sha256": { "attention_branch_output": "d7913db2408d5062b288c4b95e9ffb2f0d06e58bee52e81d9300885e31436501", "mlp_branch_output": "490a3ed9260ad0b92a1839e225a59c197ccf4984373e834d9df6ed56c379de39", "post_attention_state": "40fd0d817b1a8fb04957cc5ff2887bc65937f1776b53a68242e4c7341723dda7", "post_mlp_state": "79ad652a3aa7b0850b826f149977b0495d569b726bb8d66c66c52638215ffd53", "pre_attention_input": "0db2d20deb5c148ef675d303417844641f9632a980cb7b1bc2b39e8bf6c514f2", "pre_mlp_input": "f2daf0e86334d46ef8de0e95de81ea9f5fd3989918e22ce51c09039c50c57cf2" }, "logits_sha256": "1a0f96c15dd52249e0f831e43c96af3ac5c3a1538f81e995c2942a1eb3edcccc", "loss_nats": 2.120351791381836, "mixer_summary_sha256": "c3dc8918354fb21100a845676f3328608d15f24ff7f23ed2830e4f5211e0fe8d" }, "loss_nats": 2.120351791381836, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6572662591934204, "group": 1, "latest_source_mass": 0.6088608503341675, "layer": 1, "max_source_mass": 0.6088608503341675, "mixer_index": 1, "normalized_entropy": 0.9482347726819877, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3911391496658325, "median": 0.3669866621494293, "p05": 0.30033501982688904, "p95": 0.5365557670593262 }, { "label": "current_group_partial", "mean": 0.6088608503341675, "median": 0.6330133676528931, "p05": 0.4634442627429962, "p95": 0.6996649503707886 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6751130819320679, "group": 1, "latest_source_mass": 0.5585282444953918, "layer": 2, "max_source_mass": 0.5585282444953918, "mixer_index": 2, "normalized_entropy": 0.9739822953426588, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.44147175550460815, "median": 0.4214625358581543, "p05": 0.3529680669307709, "p95": 0.5934914946556091 }, { "label": "current_group_partial", "mean": 0.5585282444953918, "median": 0.5785374641418457, "p05": 0.40650850534439087, "p95": 0.6470319032669067 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6885842084884644, "group": 1, "latest_source_mass": 0.5103577375411987, "layer": 2, "max_source_mass": 0.5103577375411987, "mixer_index": 3, "normalized_entropy": 0.9934170228207596, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4896422028541565, "median": 0.48023492097854614, "p05": 0.42685163021087646, "p95": 0.5744246244430542 }, { "label": "current_group_partial", "mean": 0.5103577375411987, "median": 0.5197650790214539, "p05": 0.4255753755569458, "p95": 0.5731483697891235 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.668085515499115, "group": 1, "latest_source_mass": 0.5667530298233032, "layer": 3, "max_source_mass": 0.5667530298233032, "mixer_index": 4, "normalized_entropy": 0.96384366010032, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4332469701766968, "median": 0.4057430326938629, "p05": 0.3270769417285919, "p95": 0.6122807264328003 }, { "label": "current_group_partial", "mean": 0.5667530298233032, "median": 0.5942569375038147, "p05": 0.3877193331718445, "p95": 0.6729230880737305 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6886032223701477, "group": 1, "latest_source_mass": 0.4881485104560852, "layer": 3, "max_source_mass": 0.5118515491485596, "mixer_index": 5, "normalized_entropy": 0.9934444540535723, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5118515491485596, "median": 0.5089530944824219, "p05": 0.44159775972366333, "p95": 0.5947571992874146 }, { "label": "current_group_partial", "mean": 0.4881485104560852, "median": 0.4910469055175781, "p05": 0.40524277091026306, "p95": 0.5584021806716919 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6789880990982056, "group": 1, "latest_source_mass": 0.5300825834274292, "layer": 4, "max_source_mass": 0.5300825834274292, "mixer_index": 6, "normalized_entropy": 0.9795727633916053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4699174761772156, "median": 0.45400309562683105, "p05": 0.3678591251373291, "p95": 0.6229325532913208 }, { "label": "current_group_partial", "mean": 0.5300825834274292, "median": 0.545996904373169, "p05": 0.3770674467086792, "p95": 0.6321409344673157 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6772009134292603, "group": 1, "latest_source_mass": 0.5604768991470337, "layer": 4, "max_source_mass": 0.5604768991470337, "mixer_index": 7, "normalized_entropy": 0.97699439948987, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4395230710506439, "median": 0.4313446283340454, "p05": 0.34554681181907654, "p95": 0.5607337951660156 }, { "label": "current_group_partial", "mean": 0.5604768991470337, "median": 0.5686553716659546, "p05": 0.4392662048339844, "p95": 0.6544531583786011 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6781555414199829, "group": 2, "latest_source_mass": 0.5458660125732422, "layer": 5, "max_source_mass": 0.5458660125732422, "mixer_index": 8, "normalized_entropy": 0.9783716365579794, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4541339576244354, "median": 0.4442256689071655, "p05": 0.3537072241306305, "p95": 0.5944923758506775 }, { "label": "completed_group_1", "mean": 0.5458660125732422, "median": 0.5557742714881897, "p05": 0.40550756454467773, "p95": 0.6462928056716919 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0917184352874756, "group": 2, "latest_source_mass": 0.33584368228912354, "layer": 5, "max_source_mass": 0.355538010597229, "mixer_index": 9, "normalized_entropy": 0.9937249442303328, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.30861833691596985, "median": 0.3010689616203308, "p05": 0.2663846015930176, "p95": 0.37415313720703125 }, { "label": "completed_group_1", "mean": 0.355538010597229, "median": 0.3584616184234619, "p05": 0.2822456955909729, "p95": 0.4176298975944519 }, { "label": "current_group_partial", "mean": 0.33584368228912354, "median": 0.3367723822593689, "p05": 0.30035722255706787, "p95": 0.37120699882507324 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0860013961791992, "group": 2, "latest_source_mass": 0.37406760454177856, "layer": 6, "max_source_mass": 0.37406760454177856, "mixer_index": 10, "normalized_entropy": 0.9885210709738198, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30104559659957886, "median": 0.28779512643814087, "p05": 0.24668964743614197, "p95": 0.39145994186401367 }, { "label": "completed_group_1", "mean": 0.3248867988586426, "median": 0.3244113028049469, "p05": 0.2869458496570587, "p95": 0.3649960160255432 }, { "label": "current_group_partial", "mean": 0.37406760454177856, "median": 0.3894866406917572, "p05": 0.25732842087745667, "p95": 0.4404592216014862 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0827364921569824, "group": 2, "latest_source_mass": 0.378738671541214, "layer": 6, "max_source_mass": 0.378738671541214, "mixer_index": 11, "normalized_entropy": 0.9855492272616264, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2735002934932709, "median": 0.2623218297958374, "p05": 0.22696587443351746, "p95": 0.3791309595108032 }, { "label": "completed_group_1", "mean": 0.34776100516319275, "median": 0.34822437167167664, "p05": 0.2800303101539612, "p95": 0.40881502628326416 }, { "label": "current_group_partial", "mean": 0.378738671541214, "median": 0.3773481547832489, "p05": 0.3319735825061798, "p95": 0.42474985122680664 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0724949836730957, "group": 2, "latest_source_mass": 0.40881654620170593, "layer": 7, "max_source_mass": 0.40881654620170593, "mixer_index": 12, "normalized_entropy": 0.9762270044997612, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2886848449707031, "median": 0.2789040505886078, "p05": 0.2284647673368454, "p95": 0.38948193192481995 }, { "label": "completed_group_1", "mean": 0.30249863862991333, "median": 0.29331475496292114, "p05": 0.26091599464416504, "p95": 0.38118377327919006 }, { "label": "current_group_partial", "mean": 0.40881654620170593, "median": 0.43531113862991333, "p05": 0.24153971672058105, "p95": 0.48550277948379517 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0897927284240723, "group": 2, "latest_source_mass": 0.3727659583091736, "layer": 7, "max_source_mass": 0.3727659583091736, "mixer_index": 13, "normalized_entropy": 0.9919720903042785, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2973601222038269, "median": 0.28749629855155945, "p05": 0.25842589139938354, "p95": 0.36914706230163574 }, { "label": "completed_group_1", "mean": 0.3298739790916443, "median": 0.32824352383613586, "p05": 0.2834911346435547, "p95": 0.37481051683425903 }, { "label": "current_group_partial", "mean": 0.3727659583091736, "median": 0.369523286819458, "p05": 0.3233802914619446, "p95": 0.4292686879634857 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0697213411331177, "group": 2, "latest_source_mass": 0.4076794981956482, "layer": 8, "max_source_mass": 0.4076794981956482, "mixer_index": 14, "normalized_entropy": 0.9737023262592323, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3028595447540283, "median": 0.29215747117996216, "p05": 0.25508639216423035, "p95": 0.3769429922103882 }, { "label": "completed_group_1", "mean": 0.28946101665496826, "median": 0.268873929977417, "p05": 0.2331261932849884, "p95": 0.3979876637458801 }, { "label": "current_group_partial", "mean": 0.4076794981956482, "median": 0.4429859519004822, "p05": 0.22969894111156464, "p95": 0.4997333288192749 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0826778411865234, "group": 2, "latest_source_mass": 0.3901388645172119, "layer": 8, "max_source_mass": 0.3901388645172119, "mixer_index": 15, "normalized_entropy": 0.9854958408476349, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2862996757030487, "median": 0.2773204445838928, "p05": 0.22970792651176453, "p95": 0.39824816584587097 }, { "label": "completed_group_1", "mean": 0.3235614597797394, "median": 0.32365044951438904, "p05": 0.26518940925598145, "p95": 0.38037896156311035 }, { "label": "current_group_partial", "mean": 0.3901388645172119, "median": 0.38918495178222656, "p05": 0.3267495334148407, "p95": 0.44896650314331055 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0579445362091064, "group": 3, "latest_source_mass": 0.4269215166568756, "layer": 9, "max_source_mass": 0.4269215166568756, "mixer_index": 16, "normalized_entropy": 0.9629826164530652, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2899969816207886, "median": 0.2869234085083008, "p05": 0.22596141695976257, "p95": 0.37520572543144226 }, { "label": "completed_group_1", "mean": 0.2830814719200134, "median": 0.2585422396659851, "p05": 0.2167314738035202, "p95": 0.408857524394989 }, { "label": "completed_group_2", "mean": 0.4269215166568756, "median": 0.4538877308368683, "p05": 0.23425813019275665, "p95": 0.5443560481071472 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3809051513671875, "group": 3, "latest_source_mass": 0.2620403468608856, "layer": 9, "max_source_mass": 0.26517200469970703, "mixer_index": 17, "normalized_entropy": 0.9961125069077325, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.23795948922634125, "median": 0.23573458194732666, "p05": 0.20756788551807404, "p95": 0.2841742932796478 }, { "label": "completed_group_1", "mean": 0.2348281741142273, "median": 0.23269692063331604, "p05": 0.20418217778205872, "p95": 0.2688288688659668 }, { "label": "completed_group_2", "mean": 0.26517200469970703, "median": 0.2671319544315338, "p05": 0.22843149304389954, "p95": 0.29650676250457764 }, { "label": "current_group_partial", "mean": 0.2620403468608856, "median": 0.26025551557540894, "p05": 0.2228235900402069, "p95": 0.30686914920806885 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3605632781982422, "group": 3, "latest_source_mass": 0.27637380361557007, "layer": 10, "max_source_mass": 0.27637380361557007, "mixer_index": 18, "normalized_entropy": 0.9814389471361176, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.24186556041240692, "median": 0.23976491391658783, "p05": 0.19832007586956024, "p95": 0.3010247051715851 }, { "label": "completed_group_1", "mean": 0.22099661827087402, "median": 0.19567278027534485, "p05": 0.1509825587272644, "p95": 0.35698726773262024 }, { "label": "completed_group_2", "mean": 0.2607640027999878, "median": 0.2613059878349304, "p05": 0.2048056572675705, "p95": 0.31461119651794434 }, { "label": "current_group_partial", "mean": 0.27637380361557007, "median": 0.2970566749572754, "p05": 0.1543806940317154, "p95": 0.359146386384964 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3829561471939087, "group": 3, "latest_source_mass": 0.26782411336898804, "layer": 10, "max_source_mass": 0.26782411336898804, "mixer_index": 19, "normalized_entropy": 0.9975919876617797, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.23731297254562378, "median": 0.23403030633926392, "p05": 0.21617025136947632, "p95": 0.2765912413597107 }, { "label": "completed_group_1", "mean": 0.23829799890518188, "median": 0.23762314021587372, "p05": 0.21581488847732544, "p95": 0.26157575845718384 }, { "label": "completed_group_2", "mean": 0.2565649151802063, "median": 0.256875216960907, "p05": 0.235240638256073, "p95": 0.2770921587944031 }, { "label": "current_group_partial", "mean": 0.26782411336898804, "median": 0.2663458585739136, "p05": 0.2370651215314865, "p95": 0.3028164207935333 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.35977041721344, "group": 3, "latest_source_mass": 0.29567578434944153, "layer": 11, "max_source_mass": 0.29567578434944153, "mixer_index": 20, "normalized_entropy": 0.9808670188306733, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.23858636617660522, "median": 0.23665651679039001, "p05": 0.19799774885177612, "p95": 0.2946481704711914 }, { "label": "completed_group_1", "mean": 0.21978360414505005, "median": 0.19543592631816864, "p05": 0.158006951212883, "p95": 0.33895981311798096 }, { "label": "completed_group_2", "mean": 0.2459542453289032, "median": 0.24462920427322388, "p05": 0.21809649467468262, "p95": 0.27666720747947693 }, { "label": "current_group_partial", "mean": 0.29567578434944153, "median": 0.3254140615463257, "p05": 0.15821409225463867, "p95": 0.38422760367393494 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3790044784545898, "group": 3, "latest_source_mass": 0.27211782336235046, "layer": 11, "max_source_mass": 0.27211782336235046, "mixer_index": 21, "normalized_entropy": 0.9947414612150541, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.24452988803386688, "median": 0.24109914898872375, "p05": 0.21188999712467194, "p95": 0.29188552498817444 }, { "label": "completed_group_1", "mean": 0.2358427196741104, "median": 0.2335224598646164, "p05": 0.20158399641513824, "p95": 0.27844953536987305 }, { "label": "completed_group_2", "mean": 0.24750956892967224, "median": 0.24642077088356018, "p05": 0.21784070134162903, "p95": 0.28041350841522217 }, { "label": "current_group_partial", "mean": 0.27211782336235046, "median": 0.276747465133667, "p05": 0.20197471976280212, "p95": 0.3271600306034088 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3556556701660156, "group": 3, "latest_source_mass": 0.3048465847969055, "layer": 12, "max_source_mass": 0.3048465847969055, "mixer_index": 22, "normalized_entropy": 0.9778988562507576, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2410905957221985, "median": 0.23850542306900024, "p05": 0.20414574444293976, "p95": 0.28753191232681274 }, { "label": "completed_group_1", "mean": 0.22088873386383057, "median": 0.19732099771499634, "p05": 0.15404488146305084, "p95": 0.34053435921669006 }, { "label": "completed_group_2", "mean": 0.23317405581474304, "median": 0.231553316116333, "p05": 0.20150601863861084, "p95": 0.26888900995254517 }, { "label": "current_group_partial", "mean": 0.3048465847969055, "median": 0.3246956467628479, "p05": 0.15267115831375122, "p95": 0.41261592507362366 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3804312944412231, "group": 3, "latest_source_mass": 0.2690383195877075, "layer": 12, "max_source_mass": 0.2690383195877075, "mixer_index": 23, "normalized_entropy": 0.9957706913891425, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.26568183302879333, "median": 0.2650548219680786, "p05": 0.22915047407150269, "p95": 0.3039885461330414 }, { "label": "completed_group_1", "mean": 0.22685569524765015, "median": 0.2285763919353485, "p05": 0.19875401258468628, "p95": 0.2494552731513977 }, { "label": "completed_group_2", "mean": 0.2384241372346878, "median": 0.23920457065105438, "p05": 0.2169211506843567, "p95": 0.25894689559936523 }, { "label": "current_group_partial", "mean": 0.2690383195877075, "median": 0.27138766646385193, "p05": 0.2234886884689331, "p95": 0.312222421169281 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3533532619476318, "group": 4, "latest_source_mass": 0.3041262626647949, "layer": 13, "max_source_mass": 0.3041262626647949, "mixer_index": 24, "normalized_entropy": 0.9762380197913754, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24537263810634613, "median": 0.24389782547950745, "p05": 0.2054842710494995, "p95": 0.2901798486709595 }, { "label": "completed_group_1", "mean": 0.2171444594860077, "median": 0.1941797286272049, "p05": 0.1490350365638733, "p95": 0.338962197303772 }, { "label": "completed_group_2", "mean": 0.23335666954517365, "median": 0.23076492547988892, "p05": 0.20317009091377258, "p95": 0.27066874504089355 }, { "label": "completed_group_3", "mean": 0.3041262626647949, "median": 0.3265917897224426, "p05": 0.1414138674736023, "p95": 0.42294132709503174 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6053518056869507, "group": 4, "latest_source_mass": 0.22539189457893372, "layer": 13, "max_source_mass": 0.22539189457893372, "mixer_index": 25, "normalized_entropy": 0.9974611591316562, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20177139341831207, "median": 0.20180779695510864, "p05": 0.18717515468597412, "p95": 0.21589727699756622 }, { "label": "completed_group_1", "mean": 0.18913303315639496, "median": 0.19024527072906494, "p05": 0.17409206926822662, "p95": 0.20153141021728516 }, { "label": "completed_group_2", "mean": 0.18395330011844635, "median": 0.18139007687568665, "p05": 0.16835211217403412, "p95": 0.20520012080669403 }, { "label": "completed_group_3", "mean": 0.1997503787279129, "median": 0.19939276576042175, "p05": 0.1866363286972046, "p95": 0.2140733301639557 }, { "label": "current_group_partial", "mean": 0.22539189457893372, "median": 0.2293616533279419, "p05": 0.18638378381729126, "p95": 0.24480541050434113 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.585533618927002, "group": 4, "latest_source_mass": 0.25353068113327026, "layer": 14, "max_source_mass": 0.25353068113327026, "mixer_index": 26, "normalized_entropy": 0.9851474273580733, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19244730472564697, "median": 0.19150400161743164, "p05": 0.1739657074213028, "p95": 0.21505704522132874 }, { "label": "completed_group_1", "mean": 0.1780712604522705, "median": 0.16121169924736023, "p05": 0.1377859115600586, "p95": 0.2661179304122925 }, { "label": "completed_group_2", "mean": 0.17078758776187897, "median": 0.16340723633766174, "p05": 0.14458762109279633, "p95": 0.22640380263328552 }, { "label": "completed_group_3", "mean": 0.20516318082809448, "median": 0.20959466695785522, "p05": 0.15043368935585022, "p95": 0.2540265917778015 }, { "label": "current_group_partial", "mean": 0.25353068113327026, "median": 0.2702726125717163, "p05": 0.14368262887001038, "p95": 0.29619652032852173 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6022908687591553, "group": 4, "latest_source_mass": 0.23394522070884705, "layer": 14, "max_source_mass": 0.23394522070884705, "mixer_index": 27, "normalized_entropy": 0.9955592920859333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1985350400209427, "median": 0.1984509825706482, "p05": 0.1769370585680008, "p95": 0.22112208604812622 }, { "label": "completed_group_1", "mean": 0.18978449702262878, "median": 0.1900787055492401, "p05": 0.1711708903312683, "p95": 0.20756162703037262 }, { "label": "completed_group_2", "mean": 0.17792725563049316, "median": 0.17577432096004486, "p05": 0.1577838957309723, "p95": 0.20609596371650696 }, { "label": "completed_group_3", "mean": 0.1998080015182495, "median": 0.200008824467659, "p05": 0.18536703288555145, "p95": 0.2129323035478592 }, { "label": "current_group_partial", "mean": 0.23394522070884705, "median": 0.23855753242969513, "p05": 0.18709585070610046, "p95": 0.2655036449432373 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5833771228790283, "group": 4, "latest_source_mass": 0.2590453028678894, "layer": 15, "max_source_mass": 0.2590453028678894, "mixer_index": 28, "normalized_entropy": 0.9838075210272276, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1926397681236267, "median": 0.19182440638542175, "p05": 0.1704544872045517, "p95": 0.21726183593273163 }, { "label": "completed_group_1", "mean": 0.1801614761352539, "median": 0.16366446018218994, "p05": 0.13534875214099884, "p95": 0.27078741788864136 }, { "label": "completed_group_2", "mean": 0.17439624667167664, "median": 0.16901585459709167, "p05": 0.14859820902347565, "p95": 0.22471660375595093 }, { "label": "completed_group_3", "mean": 0.19375719130039215, "median": 0.19451642036437988, "p05": 0.16403654217720032, "p95": 0.22201430797576904 }, { "label": "current_group_partial", "mean": 0.2590453028678894, "median": 0.27778831124305725, "p05": 0.1263412982225418, "p95": 0.3228909373283386 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.601109504699707, "group": 4, "latest_source_mass": 0.23593464493751526, "layer": 15, "max_source_mass": 0.23593464493751526, "mixer_index": 29, "normalized_entropy": 0.994825269325365, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.19747966527938843, "median": 0.19566109776496887, "p05": 0.17110133171081543, "p95": 0.22994962334632874 }, { "label": "completed_group_1", "mean": 0.19028550386428833, "median": 0.1902298927307129, "p05": 0.16050578653812408, "p95": 0.22037073969841003 }, { "label": "completed_group_2", "mean": 0.18151576817035675, "median": 0.18003535270690918, "p05": 0.15826183557510376, "p95": 0.21004454791545868 }, { "label": "completed_group_3", "mean": 0.19478440284729004, "median": 0.1937907189130783, "p05": 0.17653080821037292, "p95": 0.2156463861465454 }, { "label": "current_group_partial", "mean": 0.23593464493751526, "median": 0.23639161884784698, "p05": 0.1930353045463562, "p95": 0.27262991666793823 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5755603313446045, "group": 4, "latest_source_mass": 0.27402210235595703, "layer": 16, "max_source_mass": 0.27402210235595703, "mixer_index": 30, "normalized_entropy": 0.9789506753707202, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1969345510005951, "median": 0.19588357210159302, "p05": 0.17255066335201263, "p95": 0.22505135834217072 }, { "label": "completed_group_1", "mean": 0.17712508141994476, "median": 0.1627851128578186, "p05": 0.13340646028518677, "p95": 0.26109084486961365 }, { "label": "completed_group_2", "mean": 0.16844677925109863, "median": 0.16122135519981384, "p05": 0.14004012942314148, "p95": 0.22884467244148254 }, { "label": "completed_group_3", "mean": 0.18347150087356567, "median": 0.18296413123607635, "p05": 0.16371525824069977, "p95": 0.2048267126083374 }, { "label": "current_group_partial", "mean": 0.27402210235595703, "median": 0.29705047607421875, "p05": 0.12304861843585968, "p95": 0.34848910570144653 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5980476140975952, "group": 4, "latest_source_mass": 0.24492081999778748, "layer": 16, "max_source_mass": 0.24492081999778748, "mixer_index": 31, "normalized_entropy": 0.9929228097284731, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19043892621994019, "median": 0.18689709901809692, "p05": 0.16917146742343903, "p95": 0.22622565925121307 }, { "label": "completed_group_1", "mean": 0.1923028826713562, "median": 0.19136476516723633, "p05": 0.1641806662082672, "p95": 0.22264797985553741 }, { "label": "completed_group_2", "mean": 0.1810823678970337, "median": 0.1799810826778412, "p05": 0.1545257568359375, "p95": 0.21300536394119263 }, { "label": "completed_group_3", "mean": 0.19125500321388245, "median": 0.19091501832008362, "p05": 0.16802990436553955, "p95": 0.21442335844039917 }, { "label": "current_group_partial", "mean": 0.24492081999778748, "median": 0.24817338585853577, "p05": 0.19009609520435333, "p95": 0.29222598671913147 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5842485427856445, "group": 5, "latest_source_mass": 0.26211416721343994, "layer": 17, "max_source_mass": 0.26211416721343994, "mixer_index": 32, "normalized_entropy": 0.9843489646578789, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1964084655046463, "median": 0.1943213939666748, "p05": 0.1771012842655182, "p95": 0.22198832035064697 }, { "label": "completed_group_1", "mean": 0.17743000388145447, "median": 0.16406384110450745, "p05": 0.13566289842128754, "p95": 0.25637227296829224 }, { "label": "completed_group_2", "mean": 0.17621049284934998, "median": 0.1722719371318817, "p05": 0.15086707472801208, "p95": 0.22157853841781616 }, { "label": "completed_group_3", "mean": 0.1878368854522705, "median": 0.18780654668807983, "p05": 0.16647562384605408, "p95": 0.20967811346054077 }, { "label": "completed_group_4", "mean": 0.26211416721343994, "median": 0.27968263626098633, "p05": 0.1302976906299591, "p95": 0.32829993963241577 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7856926918029785, "group": 5, "latest_source_mass": 0.18537181615829468, "layer": 17, "max_source_mass": 0.18537181615829468, "mixer_index": 33, "normalized_entropy": 0.9966140670501437, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15997162461280823, "median": 0.15724682807922363, "p05": 0.13894931972026825, "p95": 0.18793770670890808 }, { "label": "completed_group_1", "mean": 0.16431674361228943, "median": 0.1631746143102646, "p05": 0.1443376988172531, "p95": 0.18849408626556396 }, { "label": "completed_group_2", "mean": 0.15234553813934326, "median": 0.15064480900764465, "p05": 0.13384775817394257, "p95": 0.17680883407592773 }, { "label": "completed_group_3", "mean": 0.15752777457237244, "median": 0.15624287724494934, "p05": 0.1438755989074707, "p95": 0.17557287216186523 }, { "label": "completed_group_4", "mean": 0.18046650290489197, "median": 0.18079496920108795, "p05": 0.1579463630914688, "p95": 0.20150543749332428 }, { "label": "current_group_partial", "mean": 0.18537181615829468, "median": 0.18655891716480255, "p05": 0.15393848717212677, "p95": 0.2133883237838745 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7751184701919556, "group": 5, "latest_source_mass": 0.2034376710653305, "layer": 18, "max_source_mass": 0.2034376710653305, "mixer_index": 34, "normalized_entropy": 0.9907124816015239, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16064734756946564, "median": 0.1604275405406952, "p05": 0.135296031832695, "p95": 0.1862303614616394 }, { "label": "completed_group_1", "mean": 0.15155573189258575, "median": 0.14294931292533875, "p05": 0.12762469053268433, "p95": 0.20150375366210938 }, { "label": "completed_group_2", "mean": 0.14907032251358032, "median": 0.14577648043632507, "p05": 0.13263380527496338, "p95": 0.1819881796836853 }, { "label": "completed_group_3", "mean": 0.14832918345928192, "median": 0.1450541615486145, "p05": 0.12743958830833435, "p95": 0.18020303547382355 }, { "label": "completed_group_4", "mean": 0.18695974349975586, "median": 0.19112101197242737, "p05": 0.14380104839801788, "p95": 0.21154803037643433 }, { "label": "current_group_partial", "mean": 0.2034376710653305, "median": 0.2150307595729828, "p05": 0.12007137387990952, "p95": 0.24437281489372253 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7829411029815674, "group": 5, "latest_source_mass": 0.1976606249809265, "layer": 18, "max_source_mass": 0.1976606249809265, "mixer_index": 35, "normalized_entropy": 0.9950783760890145, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14932186901569366, "median": 0.14249257743358612, "p05": 0.12489590048789978, "p95": 0.19243758916854858 }, { "label": "completed_group_1", "mean": 0.16724151372909546, "median": 0.1661989390850067, "p05": 0.1489294171333313, "p95": 0.186150923371315 }, { "label": "completed_group_2", "mean": 0.15592648088932037, "median": 0.15542590618133545, "p05": 0.13526445627212524, "p95": 0.1783086359500885 }, { "label": "completed_group_3", "mean": 0.1555134654045105, "median": 0.154983788728714, "p05": 0.14075525104999542, "p95": 0.17211037874221802 }, { "label": "completed_group_4", "mean": 0.17433607578277588, "median": 0.17505210638046265, "p05": 0.15327315032482147, "p95": 0.19426627457141876 }, { "label": "current_group_partial", "mean": 0.1976606249809265, "median": 0.20035341382026672, "p05": 0.15607678890228271, "p95": 0.22393837571144104 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7661566734313965, "group": 5, "latest_source_mass": 0.22923919558525085, "layer": 19, "max_source_mass": 0.22923919558525085, "mixer_index": 36, "normalized_entropy": 0.9857108075964633, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.16592846810817719, "median": 0.16519835591316223, "p05": 0.13959640264511108, "p95": 0.1943950057029724 }, { "label": "completed_group_1", "mean": 0.143815815448761, "median": 0.13836468756198883, "p05": 0.12640488147735596, "p95": 0.17898298799991608 }, { "label": "completed_group_2", "mean": 0.1375015527009964, "median": 0.13198094069957733, "p05": 0.11646819114685059, "p95": 0.1853182464838028 }, { "label": "completed_group_3", "mean": 0.144493967294693, "median": 0.14179405570030212, "p05": 0.1222185343503952, "p95": 0.17581477761268616 }, { "label": "completed_group_4", "mean": 0.17902101576328278, "median": 0.18112671375274658, "p05": 0.15200620889663696, "p95": 0.1975291222333908 }, { "label": "current_group_partial", "mean": 0.22923919558525085, "median": 0.24496859312057495, "p05": 0.13256870210170746, "p95": 0.270551860332489 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7807259559631348, "group": 5, "latest_source_mass": 0.20281299948692322, "layer": 19, "max_source_mass": 0.20281299948692322, "mixer_index": 37, "normalized_entropy": 0.9938420789986538, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14958029985427856, "median": 0.14521853625774384, "p05": 0.12654969096183777, "p95": 0.20142483711242676 }, { "label": "completed_group_1", "mean": 0.1695217788219452, "median": 0.16831448674201965, "p05": 0.1474662870168686, "p95": 0.19261527061462402 }, { "label": "completed_group_2", "mean": 0.15695258975028992, "median": 0.15581071376800537, "p05": 0.13313542306423187, "p95": 0.18386399745941162 }, { "label": "completed_group_3", "mean": 0.1499582827091217, "median": 0.14832203090190887, "p05": 0.1333591639995575, "p95": 0.17367161810398102 }, { "label": "completed_group_4", "mean": 0.1711740493774414, "median": 0.17146730422973633, "p05": 0.15243321657180786, "p95": 0.18792594969272614 }, { "label": "current_group_partial", "mean": 0.20281299948692322, "median": 0.2034434974193573, "p05": 0.15488654375076294, "p95": 0.2422361671924591 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.781292200088501, "group": 5, "latest_source_mass": 0.20378397405147552, "layer": 20, "max_source_mass": 0.20378397405147552, "mixer_index": 38, "normalized_entropy": 0.994158105862243, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1682473123073578, "median": 0.17088356614112854, "p05": 0.1362982541322708, "p95": 0.19248411059379578 }, { "label": "completed_group_1", "mean": 0.15929323434829712, "median": 0.15772488713264465, "p05": 0.13811591267585754, "p95": 0.1835828274488449 }, { "label": "completed_group_2", "mean": 0.14874720573425293, "median": 0.14682769775390625, "p05": 0.1326565444469452, "p95": 0.17428536713123322 }, { "label": "completed_group_3", "mean": 0.15058940649032593, "median": 0.14999014139175415, "p05": 0.13366414606571198, "p95": 0.16969719529151917 }, { "label": "completed_group_4", "mean": 0.1693388670682907, "median": 0.16931521892547607, "p05": 0.15269556641578674, "p95": 0.18495050072669983 }, { "label": "current_group_partial", "mean": 0.20378397405147552, "median": 0.21180889010429382, "p05": 0.14455918967723846, "p95": 0.23620733618736267 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7818934917449951, "group": 5, "latest_source_mass": 0.1928463578224182, "layer": 20, "max_source_mass": 0.1928463578224182, "mixer_index": 39, "normalized_entropy": 0.9944936931253889, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14812925457954407, "median": 0.14193585515022278, "p05": 0.11988933384418488, "p95": 0.20610599219799042 }, { "label": "completed_group_1", "mean": 0.18386004865169525, "median": 0.18534164130687714, "p05": 0.1614244282245636, "p95": 0.20129990577697754 }, { "label": "completed_group_2", "mean": 0.1664430797100067, "median": 0.16720223426818848, "p05": 0.14779648184776306, "p95": 0.1832200437784195 }, { "label": "completed_group_3", "mean": 0.15062816441059113, "median": 0.14718246459960938, "p05": 0.13040047883987427, "p95": 0.18431158363819122 }, { "label": "completed_group_4", "mean": 0.15809309482574463, "median": 0.15818026661872864, "p05": 0.14130482077598572, "p95": 0.17517155408859253 }, { "label": "current_group_partial", "mean": 0.1928463578224182, "median": 0.1947423219680786, "p05": 0.1518069952726364, "p95": 0.22260980308055878 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.77958083152771, "group": 6, "latest_source_mass": 0.19859939813613892, "layer": 21, "max_source_mass": 0.19859939813613892, "mixer_index": 40, "normalized_entropy": 0.9932029728825198, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.19125157594680786, "median": 0.19011858105659485, "p05": 0.15831239521503448, "p95": 0.2248726487159729 }, { "label": "completed_group_1", "mean": 0.15093469619750977, "median": 0.15087208151817322, "p05": 0.12483120709657669, "p95": 0.17756037414073944 }, { "label": "completed_group_2", "mean": 0.14643779397010803, "median": 0.14498162269592285, "p05": 0.12900538742542267, "p95": 0.16938214004039764 }, { "label": "completed_group_3", "mean": 0.1490234136581421, "median": 0.1482788324356079, "p05": 0.12938444316387177, "p95": 0.17225822806358337 }, { "label": "completed_group_4", "mean": 0.16375310719013214, "median": 0.16389217972755432, "p05": 0.14641568064689636, "p95": 0.1809726506471634 }, { "label": "completed_group_5", "mean": 0.19859939813613892, "median": 0.20098775625228882, "p05": 0.15739423036575317, "p95": 0.23065634071826935 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9289655685424805, "group": 6, "latest_source_mass": 0.19567109644412994, "layer": 21, "max_source_mass": 0.19567109644412994, "mixer_index": 41, "normalized_entropy": 0.9912922081623045, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12320417165756226, "median": 0.121359683573246, "p05": 0.09729336947202682, "p95": 0.16812030971050262 }, { "label": "completed_group_1", "mean": 0.15015369653701782, "median": 0.1501747965812683, "p05": 0.13342028856277466, "p95": 0.16470867395401 }, { "label": "completed_group_2", "mean": 0.13329030573368073, "median": 0.13304202258586884, "p05": 0.11676464229822159, "p95": 0.14988435804843903 }, { "label": "completed_group_3", "mean": 0.12274879962205887, "median": 0.12164658308029175, "p05": 0.10655395686626434, "p95": 0.1420663595199585 }, { "label": "completed_group_4", "mean": 0.13076716661453247, "median": 0.13035811483860016, "p05": 0.11734984070062637, "p95": 0.14464935660362244 }, { "label": "completed_group_5", "mean": 0.1441647708415985, "median": 0.14637865126132965, "p05": 0.11776107549667358, "p95": 0.16295388340950012 }, { "label": "current_group_partial", "mean": 0.19567109644412994, "median": 0.19740021228790283, "p05": 0.16954219341278076, "p95": 0.21879082918167114 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9340521097183228, "group": 6, "latest_source_mass": 0.18921294808387756, "layer": 22, "max_source_mass": 0.18921294808387756, "mixer_index": 42, "normalized_entropy": 0.9939061732409653, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14550700783729553, "median": 0.14537246525287628, "p05": 0.13202586770057678, "p95": 0.16015800833702087 }, { "label": "completed_group_1", "mean": 0.14042077958583832, "median": 0.13958397507667542, "p05": 0.1146586537361145, "p95": 0.1715676635503769 }, { "label": "completed_group_2", "mean": 0.13418063521385193, "median": 0.1339511275291443, "p05": 0.1221425011754036, "p95": 0.14647553861141205 }, { "label": "completed_group_3", "mean": 0.1260203719139099, "median": 0.1256953775882721, "p05": 0.11505919694900513, "p95": 0.13822400569915771 }, { "label": "completed_group_4", "mean": 0.12906625866889954, "median": 0.12889152765274048, "p05": 0.11804929375648499, "p95": 0.14088976383209229 }, { "label": "completed_group_5", "mean": 0.1355919986963272, "median": 0.13481314480304718, "p05": 0.12486162781715393, "p95": 0.1482546031475067 }, { "label": "current_group_partial", "mean": 0.18921294808387756, "median": 0.1889541745185852, "p05": 0.1605755239725113, "p95": 0.21719029545783997 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9230798482894897, "group": 6, "latest_source_mass": 0.20240548253059387, "layer": 22, "max_source_mass": 0.20240548253059387, "mixer_index": 43, "normalized_entropy": 0.9882675462806405, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12067099660634995, "median": 0.1175689548254013, "p05": 0.09146378189325333, "p95": 0.17015382647514343 }, { "label": "completed_group_1", "mean": 0.15888366103172302, "median": 0.15990646183490753, "p05": 0.13809531927108765, "p95": 0.17797793447971344 }, { "label": "completed_group_2", "mean": 0.14002521336078644, "median": 0.14027830958366394, "p05": 0.12228793650865555, "p95": 0.15711908042430878 }, { "label": "completed_group_3", "mean": 0.12021137773990631, "median": 0.11844314634799957, "p05": 0.1043848991394043, "p95": 0.14366739988327026 }, { "label": "completed_group_4", "mean": 0.12209220230579376, "median": 0.12163396179676056, "p05": 0.10983633995056152, "p95": 0.13694660365581512 }, { "label": "completed_group_5", "mean": 0.13571105897426605, "median": 0.13670915365219116, "p05": 0.11317556351423264, "p95": 0.15392079949378967 }, { "label": "current_group_partial", "mean": 0.20240548253059387, "median": 0.20184719562530518, "p05": 0.1681356430053711, "p95": 0.24128922820091248 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9308109283447266, "group": 6, "latest_source_mass": 0.1935279220342636, "layer": 23, "max_source_mass": 0.1935279220342636, "mixer_index": 44, "normalized_entropy": 0.9922405355057545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1450628638267517, "median": 0.14652788639068604, "p05": 0.12564095854759216, "p95": 0.16459055244922638 }, { "label": "completed_group_1", "mean": 0.14527733623981476, "median": 0.14751458168029785, "p05": 0.11515176296234131, "p95": 0.17206212878227234 }, { "label": "completed_group_2", "mean": 0.13644321262836456, "median": 0.13537836074829102, "p05": 0.1257452517747879, "p95": 0.15082380175590515 }, { "label": "completed_group_3", "mean": 0.12497065216302872, "median": 0.12392216920852661, "p05": 0.11321288347244263, "p95": 0.14013291895389557 }, { "label": "completed_group_4", "mean": 0.12634633481502533, "median": 0.12607431411743164, "p05": 0.11311027407646179, "p95": 0.14066654443740845 }, { "label": "completed_group_5", "mean": 0.1283716857433319, "median": 0.1277504414319992, "p05": 0.11627928167581558, "p95": 0.14233072102069855 }, { "label": "current_group_partial", "mean": 0.1935279220342636, "median": 0.19237324595451355, "p05": 0.1571470946073532, "p95": 0.23567645251750946 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9273775815963745, "group": 6, "latest_source_mass": 0.18937146663665771, "layer": 23, "max_source_mass": 0.18937146663665771, "mixer_index": 45, "normalized_entropy": 0.9904761443029958, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1260491907596588, "median": 0.12174050509929657, "p05": 0.10315698385238647, "p95": 0.16765159368515015 }, { "label": "completed_group_1", "mean": 0.16657932102680206, "median": 0.16951650381088257, "p05": 0.13692717254161835, "p95": 0.18870727717876434 }, { "label": "completed_group_2", "mean": 0.14038510620594025, "median": 0.13948792219161987, "p05": 0.12602533400058746, "p95": 0.15653200447559357 }, { "label": "completed_group_3", "mean": 0.1219596415758133, "median": 0.12005066871643066, "p05": 0.10611658543348312, "p95": 0.1454348862171173 }, { "label": "completed_group_4", "mean": 0.12248437106609344, "median": 0.12224026769399643, "p05": 0.10896626859903336, "p95": 0.13705895841121674 }, { "label": "completed_group_5", "mean": 0.13317090272903442, "median": 0.13336879014968872, "p05": 0.11660954356193542, "p95": 0.14904874563217163 }, { "label": "current_group_partial", "mean": 0.18937146663665771, "median": 0.18636475503444672, "p05": 0.15230347216129303, "p95": 0.23292618989944458 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.918284296989441, "group": 6, "latest_source_mass": 0.20540057122707367, "layer": 24, "max_source_mass": 0.20540057122707367, "mixer_index": 46, "normalized_entropy": 0.9858031204167963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13994047045707703, "median": 0.14368271827697754, "p05": 0.10786622762680054, "p95": 0.16540095210075378 }, { "label": "completed_group_1", "mean": 0.15779414772987366, "median": 0.1629778891801834, "p05": 0.10690005868673325, "p95": 0.19786833226680756 }, { "label": "completed_group_2", "mean": 0.13759920001029968, "median": 0.1359691321849823, "p05": 0.12253671139478683, "p95": 0.15824976563453674 }, { "label": "completed_group_3", "mean": 0.11949826776981354, "median": 0.117921382188797, "p05": 0.10491804778575897, "p95": 0.13834650814533234 }, { "label": "completed_group_4", "mean": 0.11786683648824692, "median": 0.11655203253030777, "p05": 0.1013876423239708, "p95": 0.13959234952926636 }, { "label": "completed_group_5", "mean": 0.12190047651529312, "median": 0.12099424004554749, "p05": 0.10532263666391373, "p95": 0.1426703929901123 }, { "label": "current_group_partial", "mean": 0.20540057122707367, "median": 0.20053882896900177, "p05": 0.15279710292816162, "p95": 0.27538907527923584 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9060933589935303, "group": 6, "latest_source_mass": 0.19080331921577454, "layer": 24, "max_source_mass": 0.19080331921577454, "mixer_index": 47, "normalized_entropy": 0.9795382175887654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11515207588672638, "median": 0.108605295419693, "p05": 0.07161381095647812, "p95": 0.19197940826416016 }, { "label": "completed_group_1", "mean": 0.18610318005084991, "median": 0.1929539144039154, "p05": 0.13255266845226288, "p95": 0.22670131921768188 }, { "label": "completed_group_2", "mean": 0.15217435359954834, "median": 0.15105080604553223, "p05": 0.12881332635879517, "p95": 0.17896413803100586 }, { "label": "completed_group_3", "mean": 0.11993157863616943, "median": 0.11775963753461838, "p05": 0.09839600324630737, "p95": 0.1517515480518341 }, { "label": "completed_group_4", "mean": 0.11246243864297867, "median": 0.11158700287342072, "p05": 0.09335656464099884, "p95": 0.1348860114812851 }, { "label": "completed_group_5", "mean": 0.12337306886911392, "median": 0.12389068305492401, "p05": 0.098457470536232, "p95": 0.14636507630348206 }, { "label": "current_group_partial", "mean": 0.19080331921577454, "median": 0.18048369884490967, "p05": 0.1278817355632782, "p95": 0.28530141711235046 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9199097156524658, "group": 7, "latest_source_mass": 0.15970462560653687, "layer": 25, "max_source_mass": 0.16625350713729858, "mixer_index": 48, "normalized_entropy": 0.9866384203733816, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13630829751491547, "median": 0.14164748787879944, "p05": 0.0576973557472229, "p95": 0.20787456631660461 }, { "label": "completed_group_1", "mean": 0.16625350713729858, "median": 0.17280495166778564, "p05": 0.10803423821926117, "p95": 0.20908519625663757 }, { "label": "completed_group_2", "mean": 0.14684012532234192, "median": 0.1455259621143341, "p05": 0.12492340058088303, "p95": 0.1712140440940857 }, { "label": "completed_group_3", "mean": 0.12322510778903961, "median": 0.12119361758232117, "p05": 0.0957479178905487, "p95": 0.1575886607170105 }, { "label": "completed_group_4", "mean": 0.12947362661361694, "median": 0.12904725968837738, "p05": 0.10726519674062729, "p95": 0.1540917158126831 }, { "label": "completed_group_5", "mean": 0.1381947249174118, "median": 0.139154314994812, "p05": 0.11157024651765823, "p95": 0.1611405611038208 }, { "label": "completed_group_6", "mean": 0.15970462560653687, "median": 0.1496606469154358, "p05": 0.10531163960695267, "p95": 0.24789606034755707 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.0490760803222656, "group": 7, "latest_source_mass": 0.09463626891374588, "layer": 25, "max_source_mass": 0.1722080111503601, "mixer_index": 49, "normalized_entropy": 0.9853972998283761, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10742150247097015, "median": 0.10384121537208557, "p05": 0.07562363147735596, "p95": 0.16456669569015503 }, { "label": "completed_group_1", "mean": 0.1722080111503601, "median": 0.17752903699874878, "p05": 0.12821395695209503, "p95": 0.20726709067821503 }, { "label": "completed_group_2", "mean": 0.14153704047203064, "median": 0.14094984531402588, "p05": 0.1209082081913948, "p95": 0.16446956992149353 }, { "label": "completed_group_3", "mean": 0.1159135028719902, "median": 0.11460001766681671, "p05": 0.09666494280099869, "p95": 0.14028386771678925 }, { "label": "completed_group_4", "mean": 0.10868977010250092, "median": 0.10807925462722778, "p05": 0.09490883350372314, "p95": 0.124228335916996 }, { "label": "completed_group_5", "mean": 0.1118093952536583, "median": 0.11115369200706482, "p05": 0.0934501588344574, "p95": 0.13121479749679565 }, { "label": "completed_group_6", "mean": 0.1477845162153244, "median": 0.143342986702919, "p05": 0.10309553891420364, "p95": 0.20393821597099304 }, { "label": "current_group_partial", "mean": 0.09463626891374588, "median": 0.0896594375371933, "p05": 0.07582583278417587, "p95": 0.13509176671504974 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.051286220550537, "group": 7, "latest_source_mass": 0.10080308467149734, "layer": 26, "max_source_mass": 0.16089829802513123, "mixer_index": 50, "normalized_entropy": 0.9864601526107082, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1131708100438118, "median": 0.11772071570158005, "p05": 0.05498850345611572, "p95": 0.16554684937000275 }, { "label": "completed_group_1", "mean": 0.16089829802513123, "median": 0.16788098216056824, "p05": 0.10815531760454178, "p95": 0.1984335482120514 }, { "label": "completed_group_2", "mean": 0.14448881149291992, "median": 0.14400863647460938, "p05": 0.12385094165802002, "p95": 0.1663750410079956 }, { "label": "completed_group_3", "mean": 0.11956015229225159, "median": 0.11957795917987823, "p05": 0.09828256070613861, "p95": 0.14187976717948914 }, { "label": "completed_group_4", "mean": 0.10795272141695023, "median": 0.10684113204479218, "p05": 0.09203589707612991, "p95": 0.12730176746845245 }, { "label": "completed_group_5", "mean": 0.11352235078811646, "median": 0.1128544956445694, "p05": 0.09500996023416519, "p95": 0.1338450163602829 }, { "label": "completed_group_6", "mean": 0.13960374891757965, "median": 0.13686895370483398, "p05": 0.09854506701231003, "p95": 0.19470667839050293 }, { "label": "current_group_partial", "mean": 0.10080308467149734, "median": 0.0929388776421547, "p05": 0.07505455613136292, "p95": 0.15377722680568695 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0247201919555664, "group": 7, "latest_source_mass": 0.08503472805023193, "layer": 26, "max_source_mass": 0.1894242763519287, "mixer_index": 51, "normalized_entropy": 0.9736845933740154, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.101108118891716, "median": 0.09478609263896942, "p05": 0.06526951491832733, "p95": 0.17756468057632446 }, { "label": "completed_group_1", "mean": 0.1894242763519287, "median": 0.19696936011314392, "p05": 0.12617433071136475, "p95": 0.23825061321258545 }, { "label": "completed_group_2", "mean": 0.1531166434288025, "median": 0.15302017331123352, "p05": 0.11950686573982239, "p95": 0.18814897537231445 }, { "label": "completed_group_3", "mean": 0.11718504130840302, "median": 0.11499293893575668, "p05": 0.09171593189239502, "p95": 0.15106697380542755 }, { "label": "completed_group_4", "mean": 0.10284576565027237, "median": 0.10174860805273056, "p05": 0.08644768595695496, "p95": 0.1233588457107544 }, { "label": "completed_group_5", "mean": 0.10429200530052185, "median": 0.10377363860607147, "p05": 0.08251910656690598, "p95": 0.1272168904542923 }, { "label": "completed_group_6", "mean": 0.14699342846870422, "median": 0.14241871237754822, "p05": 0.09243378043174744, "p95": 0.21625562012195587 }, { "label": "current_group_partial", "mean": 0.08503472805023193, "median": 0.07344722747802734, "p05": 0.06163846701383591, "p95": 0.15428426861763 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.047537326812744, "group": 7, "latest_source_mass": 0.09834913909435272, "layer": 27, "max_source_mass": 0.16533835232257843, "mixer_index": 52, "normalized_entropy": 0.9846573158092636, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111423671245575, "median": 0.11424723267555237, "p05": 0.05927120894193649, "p95": 0.1600918024778366 }, { "label": "completed_group_1", "mean": 0.16533835232257843, "median": 0.17344561219215393, "p05": 0.10133285820484161, "p95": 0.21326853334903717 }, { "label": "completed_group_2", "mean": 0.14553974568843842, "median": 0.14541476964950562, "p05": 0.11970965564250946, "p95": 0.17045748233795166 }, { "label": "completed_group_3", "mean": 0.12043843418359756, "median": 0.12007981538772583, "p05": 0.09830569475889206, "p95": 0.14429020881652832 }, { "label": "completed_group_4", "mean": 0.10876711457967758, "median": 0.10606391727924347, "p05": 0.08962530642747879, "p95": 0.13553042709827423 }, { "label": "completed_group_5", "mean": 0.11067016422748566, "median": 0.10974282026290894, "p05": 0.09159430861473083, "p95": 0.13197509944438934 }, { "label": "completed_group_6", "mean": 0.13975468277931213, "median": 0.13449960947036743, "p05": 0.10173755139112473, "p95": 0.2017771154642105 }, { "label": "current_group_partial", "mean": 0.09834913909435272, "median": 0.09059631079435349, "p05": 0.07406806200742722, "p95": 0.15292926132678986 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0282247066497803, "group": 7, "latest_source_mass": 0.08871228992938995, "layer": 27, "max_source_mass": 0.18681631982326508, "mixer_index": 53, "normalized_entropy": 0.9753699086973703, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.100978784263134, "median": 0.0939556136727333, "p05": 0.06883562356233597, "p95": 0.17509761452674866 }, { "label": "completed_group_1", "mean": 0.18681631982326508, "median": 0.19304953515529633, "p05": 0.12605154514312744, "p95": 0.2314973771572113 }, { "label": "completed_group_2", "mean": 0.15267829596996307, "median": 0.15211889147758484, "p05": 0.11771412938833237, "p95": 0.18910512328147888 }, { "label": "completed_group_3", "mean": 0.11643697321414948, "median": 0.11495602130889893, "p05": 0.08927997201681137, "p95": 0.14835557341575623 }, { "label": "completed_group_4", "mean": 0.10204558074474335, "median": 0.10098521411418915, "p05": 0.08531612157821655, "p95": 0.12325739860534668 }, { "label": "completed_group_5", "mean": 0.10493150353431702, "median": 0.10501007735729218, "p05": 0.08253021538257599, "p95": 0.12569722533226013 }, { "label": "completed_group_6", "mean": 0.14740024507045746, "median": 0.14431840181350708, "p05": 0.09808903187513351, "p95": 0.2124406099319458 }, { "label": "current_group_partial", "mean": 0.08871228992938995, "median": 0.07655761390924454, "p05": 0.06286359578371048, "p95": 0.16396106779575348 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.053830623626709, "group": 7, "latest_source_mass": 0.11169404536485672, "layer": 28, "max_source_mass": 0.1550440490245819, "mixer_index": 54, "normalized_entropy": 0.9876837518440468, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11781683564186096, "median": 0.12262599170207977, "p05": 0.05666963383555412, "p95": 0.16679181158542633 }, { "label": "completed_group_1", "mean": 0.1550440490245819, "median": 0.1601896733045578, "p05": 0.09854641556739807, "p95": 0.19885312020778656 }, { "label": "completed_group_2", "mean": 0.1398259699344635, "median": 0.1393846720457077, "p05": 0.12018831819295883, "p95": 0.16097646951675415 }, { "label": "completed_group_3", "mean": 0.11913558095693588, "median": 0.1185217946767807, "p05": 0.09514695405960083, "p95": 0.14609722793102264 }, { "label": "completed_group_4", "mean": 0.10805961489677429, "median": 0.10712219774723053, "p05": 0.08963555097579956, "p95": 0.12859195470809937 }, { "label": "completed_group_5", "mean": 0.11164885014295578, "median": 0.11060681939125061, "p05": 0.09239380061626434, "p95": 0.13463559746742249 }, { "label": "completed_group_6", "mean": 0.13677509129047394, "median": 0.13695205748081207, "p05": 0.09874161332845688, "p95": 0.17497648298740387 }, { "label": "current_group_partial", "mean": 0.11169404536485672, "median": 0.10361628234386444, "p05": 0.07534734159708023, "p95": 0.17376387119293213 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.017721176147461, "group": 7, "latest_source_mass": 0.08305738866329193, "layer": 28, "max_source_mass": 0.19308976829051971, "mixer_index": 55, "normalized_entropy": 0.9703187782415296, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09720280766487122, "median": 0.08876170963048935, "p05": 0.0643138587474823, "p95": 0.18079209327697754 }, { "label": "completed_group_1", "mean": 0.19308976829051971, "median": 0.19938363134860992, "p05": 0.1292668581008911, "p95": 0.24086137115955353 }, { "label": "completed_group_2", "mean": 0.16317343711853027, "median": 0.1630440354347229, "p05": 0.12323655933141708, "p95": 0.20307078957557678 }, { "label": "completed_group_3", "mean": 0.11856385320425034, "median": 0.11627572774887085, "p05": 0.09080541133880615, "p95": 0.15461181104183197 }, { "label": "completed_group_4", "mean": 0.10023905336856842, "median": 0.09881964325904846, "p05": 0.08244071900844574, "p95": 0.1235516369342804 }, { "label": "completed_group_5", "mean": 0.10130266845226288, "median": 0.10165216773748398, "p05": 0.07800924777984619, "p95": 0.12226968258619308 }, { "label": "completed_group_6", "mean": 0.14337103068828583, "median": 0.1400463879108429, "p05": 0.08998478949069977, "p95": 0.21036171913146973 }, { "label": "current_group_partial", "mean": 0.08305738866329193, "median": 0.07284967601299286, "p05": 0.05788487195968628, "p95": 0.15074607729911804 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0608584880828857, "group": 8, "latest_source_mass": 0.1310214400291443, "layer": 29, "max_source_mass": 0.1425117403268814, "mixer_index": 56, "normalized_entropy": 0.9910634402437022, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12434183061122894, "median": 0.12991920113563538, "p05": 0.07406175881624222, "p95": 0.15820357203483582 }, { "label": "completed_group_1", "mean": 0.14232872426509857, "median": 0.14891478419303894, "p05": 0.08827720582485199, "p95": 0.17772579193115234 }, { "label": "completed_group_2", "mean": 0.13082580268383026, "median": 0.13057959079742432, "p05": 0.11437083035707474, "p95": 0.1484394371509552 }, { "label": "completed_group_3", "mean": 0.11684279143810272, "median": 0.11654500663280487, "p05": 0.09846262633800507, "p95": 0.13625562191009521 }, { "label": "completed_group_4", "mean": 0.10329097509384155, "median": 0.10113433003425598, "p05": 0.08762617409229279, "p95": 0.12809985876083374 }, { "label": "completed_group_5", "mean": 0.10883669555187225, "median": 0.10607670247554779, "p05": 0.08987759053707123, "p95": 0.13684110343456268 }, { "label": "completed_group_6", "mean": 0.1425117403268814, "median": 0.14483797550201416, "p05": 0.10676668584346771, "p95": 0.16993491351604462 }, { "label": "completed_group_7", "mean": 0.1310214400291443, "median": 0.12577015161514282, "p05": 0.09179449081420898, "p95": 0.18496999144554138 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.142296314239502, "group": 8, "latest_source_mass": 0.12039357423782349, "layer": 29, "max_source_mass": 0.16583400964736938, "mixer_index": 57, "normalized_entropy": 0.9750010701394441, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08636108040809631, "median": 0.08012399077415466, "p05": 0.05614824593067169, "p95": 0.1551167219877243 }, { "label": "completed_group_1", "mean": 0.16583400964736938, "median": 0.1709868609905243, "p05": 0.10593608021736145, "p95": 0.2091270387172699 }, { "label": "completed_group_2", "mean": 0.14278221130371094, "median": 0.14347383379936218, "p05": 0.10352978855371475, "p95": 0.17850933969020844 }, { "label": "completed_group_3", "mean": 0.1055997759103775, "median": 0.1052464097738266, "p05": 0.07856260240077972, "p95": 0.1339757740497589 }, { "label": "completed_group_4", "mean": 0.08843953162431717, "median": 0.08753003180027008, "p05": 0.07198000699281693, "p95": 0.1094239354133606 }, { "label": "completed_group_5", "mean": 0.08854378759860992, "median": 0.08859115839004517, "p05": 0.06814242154359818, "p95": 0.10749459266662598 }, { "label": "completed_group_6", "mean": 0.12657995522022247, "median": 0.12664274871349335, "p05": 0.08023036271333694, "p95": 0.1752617508172989 }, { "label": "completed_group_7", "mean": 0.07546606659889221, "median": 0.06708197295665741, "p05": 0.05449982360005379, "p95": 0.12993836402893066 }, { "label": "current_group_partial", "mean": 0.12039357423782349, "median": 0.11719904839992523, "p05": 0.0764557495713234, "p95": 0.17428058385849 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.191580057144165, "group": 8, "latest_source_mass": 0.10807353258132935, "layer": 30, "max_source_mass": 0.12397187203168869, "mixer_index": 58, "normalized_entropy": 0.9974310681528523, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1060912236571312, "median": 0.10528025031089783, "p05": 0.09126897901296616, "p95": 0.12398397922515869 }, { "label": "completed_group_1", "mean": 0.12397187203168869, "median": 0.12518228590488434, "p05": 0.10349762439727783, "p95": 0.14397969841957092 }, { "label": "completed_group_2", "mean": 0.11601400375366211, "median": 0.11583627760410309, "p05": 0.10074462741613388, "p95": 0.13201698660850525 }, { "label": "completed_group_3", "mean": 0.1091354638338089, "median": 0.10811793804168701, "p05": 0.09470823407173157, "p95": 0.12769103050231934 }, { "label": "completed_group_4", "mean": 0.10782144963741302, "median": 0.10718400776386261, "p05": 0.09581813216209412, "p95": 0.1218264177441597 }, { "label": "completed_group_5", "mean": 0.10360676795244217, "median": 0.1029055118560791, "p05": 0.09420947730541229, "p95": 0.11479298770427704 }, { "label": "completed_group_6", "mean": 0.12169313430786133, "median": 0.12091503292322159, "p05": 0.10930042713880539, "p95": 0.13614466786384583 }, { "label": "completed_group_7", "mean": 0.10359255224466324, "median": 0.1012670248746872, "p05": 0.09212177246809006, "p95": 0.12151782214641571 }, { "label": "current_group_partial", "mean": 0.10807353258132935, "median": 0.10744988918304443, "p05": 0.09496128559112549, "p95": 0.12403780221939087 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1432788372039795, "group": 8, "latest_source_mass": 0.08614903688430786, "layer": 30, "max_source_mass": 0.16873395442962646, "mixer_index": 59, "normalized_entropy": 0.9754482356111087, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08927534520626068, "median": 0.08175399154424667, "p05": 0.05808858573436737, "p95": 0.16859911382198334 }, { "label": "completed_group_1", "mean": 0.16873395442962646, "median": 0.17203307151794434, "p05": 0.12107796967029572, "p95": 0.2058679312467575 }, { "label": "completed_group_2", "mean": 0.14930182695388794, "median": 0.14880773425102234, "p05": 0.10963106155395508, "p95": 0.18977254629135132 }, { "label": "completed_group_3", "mean": 0.112123504281044, "median": 0.11111646890640259, "p05": 0.08233093470335007, "p95": 0.14486809074878693 }, { "label": "completed_group_4", "mean": 0.09766747057437897, "median": 0.09729938209056854, "p05": 0.08042105287313461, "p95": 0.11693428456783295 }, { "label": "completed_group_5", "mean": 0.0965762585401535, "median": 0.0965472012758255, "p05": 0.07654033601284027, "p95": 0.1171490028500557 }, { "label": "completed_group_6", "mean": 0.1252625733613968, "median": 0.12540382146835327, "p05": 0.08324124664068222, "p95": 0.1696549952030182 }, { "label": "completed_group_7", "mean": 0.07491002976894379, "median": 0.06726562976837158, "p05": 0.055054910480976105, "p95": 0.1270829290151596 }, { "label": "current_group_partial", "mean": 0.08614903688430786, "median": 0.07982201874256134, "p05": 0.06172287464141846, "p95": 0.13157080113887787 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1724743843078613, "group": 8, "latest_source_mass": 0.10534654557704926, "layer": 31, "max_source_mass": 0.12893787026405334, "mixer_index": 60, "normalized_entropy": 0.9887357017195011, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1105542704463005, "median": 0.12003511190414429, "p05": 0.041383370757102966, "p95": 0.15075469017028809 }, { "label": "completed_group_1", "mean": 0.12893787026405334, "median": 0.1339881867170334, "p05": 0.07801777124404907, "p95": 0.16926369071006775 }, { "label": "completed_group_2", "mean": 0.11523330211639404, "median": 0.1141275092959404, "p05": 0.09732259064912796, "p95": 0.1368464231491089 }, { "label": "completed_group_3", "mean": 0.10665617883205414, "median": 0.10615617036819458, "p05": 0.08223703503608704, "p95": 0.1332816481590271 }, { "label": "completed_group_4", "mean": 0.10366716235876083, "median": 0.10414719581604004, "p05": 0.08117569237947464, "p95": 0.12477125227451324 }, { "label": "completed_group_5", "mean": 0.10828493535518646, "median": 0.10714839398860931, "p05": 0.08299056440591812, "p95": 0.136210098862648 }, { "label": "completed_group_6", "mean": 0.11819782108068466, "median": 0.11870056390762329, "p05": 0.08487735688686371, "p95": 0.14779172837734222 }, { "label": "completed_group_7", "mean": 0.10312190651893616, "median": 0.09999410808086395, "p05": 0.07080516219139099, "p95": 0.1448727399110794 }, { "label": "current_group_partial", "mean": 0.10534654557704926, "median": 0.09821351617574692, "p05": 0.06854747235774994, "p95": 0.16421541571617126 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1452555656433105, "group": 8, "latest_source_mass": 0.07998920977115631, "layer": 31, "max_source_mass": 0.1701684445142746, "mixer_index": 61, "normalized_entropy": 0.9763478834940427, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09202821552753448, "median": 0.08689712733030319, "p05": 0.055717833340168, "p95": 0.16549156606197357 }, { "label": "completed_group_1", "mean": 0.1701684445142746, "median": 0.17422428727149963, "p05": 0.11712116748094559, "p95": 0.21046262979507446 }, { "label": "completed_group_2", "mean": 0.14628148078918457, "median": 0.14571070671081543, "p05": 0.11122585833072662, "p95": 0.18218384683132172 }, { "label": "completed_group_3", "mean": 0.11057832837104797, "median": 0.10866793990135193, "p05": 0.08514636754989624, "p95": 0.14279140532016754 }, { "label": "completed_group_4", "mean": 0.09616327285766602, "median": 0.09543465077877045, "p05": 0.08178231120109558, "p95": 0.1130019798874855 }, { "label": "completed_group_5", "mean": 0.09830239415168762, "median": 0.09682399034500122, "p05": 0.07916976511478424, "p95": 0.12123610824346542 }, { "label": "completed_group_6", "mean": 0.12624147534370422, "median": 0.1266379952430725, "p05": 0.08612610399723053, "p95": 0.166628897190094 }, { "label": "completed_group_7", "mean": 0.08024715632200241, "median": 0.0715465098619461, "p05": 0.05839171260595322, "p95": 0.13830669224262238 }, { "label": "current_group_partial", "mean": 0.07998920977115631, "median": 0.07393841445446014, "p05": 0.0605103000998497, "p95": 0.11983495950698853 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1762189865112305, "group": 8, "latest_source_mass": 0.09661206603050232, "layer": 32, "max_source_mass": 0.13803134858608246, "mixer_index": 62, "normalized_entropy": 0.990439943626311, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09908370673656464, "median": 0.10031558573246002, "p05": 0.06232558190822601, "p95": 0.14320872724056244 }, { "label": "completed_group_1", "mean": 0.13803134858608246, "median": 0.14098677039146423, "p05": 0.10051311552524567, "p95": 0.1691289246082306 }, { "label": "completed_group_2", "mean": 0.13144394755363464, "median": 0.13024751842021942, "p05": 0.10748467594385147, "p95": 0.15961234271526337 }, { "label": "completed_group_3", "mean": 0.11672185361385345, "median": 0.11463914811611176, "p05": 0.09377485513687134, "p95": 0.14699220657348633 }, { "label": "completed_group_4", "mean": 0.1074555367231369, "median": 0.1059759333729744, "p05": 0.08800189942121506, "p95": 0.13215802609920502 }, { "label": "completed_group_5", "mean": 0.10124591737985611, "median": 0.10008268058300018, "p05": 0.08680262416601181, "p95": 0.11920121312141418 }, { "label": "completed_group_6", "mean": 0.12084682285785675, "median": 0.11794044077396393, "p05": 0.09386572986841202, "p95": 0.15664330124855042 }, { "label": "completed_group_7", "mean": 0.08855880051851273, "median": 0.0855959877371788, "p05": 0.0724000632762909, "p95": 0.11697207391262054 }, { "label": "current_group_partial", "mean": 0.09661206603050232, "median": 0.09512065351009369, "p05": 0.07854700088500977, "p95": 0.12120820581912994 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1389729976654053, "group": 8, "latest_source_mass": 0.07340531051158905, "layer": 32, "max_source_mass": 0.17082780599594116, "mixer_index": 63, "normalized_entropy": 0.9734885635853232, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09089215844869614, "median": 0.08436901867389679, "p05": 0.05205666273832321, "p95": 0.17794299125671387 }, { "label": "completed_group_1", "mean": 0.17082780599594116, "median": 0.17341303825378418, "p05": 0.1192600280046463, "p95": 0.21146632730960846 }, { "label": "completed_group_2", "mean": 0.14892268180847168, "median": 0.14870190620422363, "p05": 0.11514535546302795, "p95": 0.18527255952358246 }, { "label": "completed_group_3", "mean": 0.11308471858501434, "median": 0.11074972152709961, "p05": 0.08665508031845093, "p95": 0.14719238877296448 }, { "label": "completed_group_4", "mean": 0.10078474879264832, "median": 0.10026010870933533, "p05": 0.08329866826534271, "p95": 0.1204671636223793 }, { "label": "completed_group_5", "mean": 0.09951519966125488, "median": 0.09887547045946121, "p05": 0.0807466059923172, "p95": 0.12053526192903519 }, { "label": "completed_group_6", "mean": 0.12574951350688934, "median": 0.12600116431713104, "p05": 0.08480837196111679, "p95": 0.17120841145515442 }, { "label": "completed_group_7", "mean": 0.07681785523891449, "median": 0.06763558089733124, "p05": 0.05441128462553024, "p95": 0.13893920183181763 }, { "label": "current_group_partial", "mean": 0.07340531051158905, "median": 0.06719464063644409, "p05": 0.054995350539684296, "p95": 0.1120883971452713 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1635239124298096, "group": null, "latest_source_mass": 0.11434720456600189, "layer": null, "max_source_mass": 0.14857198297977448, "mixer_index": null, "normalized_entropy": 0.9846621664193896, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1242082417011261, "median": 0.12387055158615112, "p05": 0.08946356177330017, "p95": 0.16199731826782227 }, { "label": "completed_group_1", "mean": 0.0880277156829834, "median": 0.08818831294775009, "p05": 0.06631279736757278, "p95": 0.10954886674880981 }, { "label": "completed_group_2", "mean": 0.0938715785741806, "median": 0.0940513014793396, "p05": 0.07511411607265472, "p95": 0.11214712262153625 }, { "label": "completed_group_3", "mean": 0.09339353442192078, "median": 0.09318573027849197, "p05": 0.07264634221792221, "p95": 0.11525531113147736 }, { "label": "completed_group_4", "mean": 0.09749531745910645, "median": 0.0970197319984436, "p05": 0.07785478234291077, "p95": 0.12001418322324753 }, { "label": "completed_group_5", "mean": 0.10569863021373749, "median": 0.10433581471443176, "p05": 0.08659540861845016, "p95": 0.1311836838722229 }, { "label": "completed_group_6", "mean": 0.13438579440116882, "median": 0.13242986798286438, "p05": 0.08274075388908386, "p95": 0.1898222714662552 }, { "label": "completed_group_7", "mean": 0.14857198297977448, "median": 0.14630873501300812, "p05": 0.11486123502254486, "p95": 0.19245916604995728 }, { "label": "completed_group_8", "mean": 0.11434720456600189, "median": 0.1131601482629776, "p05": 0.05654018372297287, "p95": 0.17325904965400696 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "c897bc93974dc10587c9be423f78a7546e14b4b7a8b97f6d5a3e52fd21bd21e8", "optimizer_state_before": "c897bc93974dc10587c9be423f78a7546e14b4b7a8b97f6d5a3e52fd21bd21e8", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "d7913db2408d5062b288c4b95e9ffb2f0d06e58bee52e81d9300885e31436501", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.01139580737799406, 0.014748631045222282, 0.012339585460722446, 0.0120800556614995, 0.01439172774553299, 0.0171960536390543, 0.01820177212357521, 0.01519495714455843, 0.016362585127353668, 0.015944689512252808, 0.017555685713887215, 0.01772458106279373, 0.025544000789523125, 0.019797103479504585, 0.02002105489373207, 0.017730148509144783, 0.022795414552092552, 0.02134518511593342, 0.021650495007634163, 0.03255617991089821, 0.033973678946495056, 0.031387828290462494, 0.019825322553515434, 0.013387269340455532, 0.015395586378872395, 0.011897368356585503, 0.014087310060858727, 0.014466723427176476, 0.006911881268024445, 0.005224450025707483, 0.00758614856749773, 0.005234629847109318 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.39030686973274e-05, "first_to_last_ratio": 5.095393144356493, "imbalance_abs_log_ratio": 1.628336826476301, "last_quartile_mean": 6.653670823197899e-06, "mean": 1.8118416377888025e-05, "normalized": [ 2.2929757071570536, 2.1131249790843154, 2.066690915233284, 1.9927541789217667, 1.8585972056365079, 1.693285810246388, 1.5304369916151304, 1.4216847468656184, 1.289196531005685, 1.185594524249146, 1.0961124019425812, 1.0141508318397905, 1.0609145912379738, 0.974788958343087, 0.8967359470227997, 0.8249692378890231, 0.8460887274678097, 0.7831360031144634, 0.7104172027135063, 0.6493842722397042, 0.8100416294668759, 0.7277529238767537, 0.6447870573653334, 0.5785188547546978, 0.45250252031967675, 0.4317242716558631, 0.41742992665867057, 0.40386477203384563, 0.31920966761853126, 0.30772885346331863, 0.30340232726708727, 0.30199743169371174 ], "peak_layer": 1, "peak_normalized": 2.2929757071570536, "population_cv": 0.5811777617331019, "rest_mean": 1.931718248602852e-05, "spike_contrast": 0.6028352945545675, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1645079393929336e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.1545088606653735e-05, 3.8286478229565546e-05, 3.744516652659513e-05, 3.610554995248094e-05, 3.367483805050142e-05, 3.067965735681355e-05, 2.7729094654205255e-05, 2.575867620180361e-05, 2.3358199541689828e-05, 2.1481095245690085e-05, 1.9859820895362645e-05, 1.8374807041254826e-05, 1.9222092305426486e-05, 1.7661632227827795e-05, 1.6247435269178823e-05, 1.4947136151022278e-05, 1.5329787856899202e-05, 1.418918418494286e-05, 1.2871634680777788e-05, 1.1765814633690752e-05, 1.4676671526103746e-05, 1.3185730495024472e-05, 1.1682520380418282e-05, 1.048184549290454e-05, 8.19862907519564e-06, 7.822160114301369e-06, 7.563169219793053e-06, 7.3173901000700425e-06, 5.7835736697597895e-06, 5.575559498538496e-06, 5.497169695445336e-06, 5.4717152124794666e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013720810875383904, "first_to_last_ratio": 5.154319968637833, "imbalance_abs_log_ratio": 1.6398351919170495, "last_quartile_mean": 2.6620021571943653e-05, "mean": 7.278344395444947e-05, "normalized": [ 2.2311752925780666, 2.1140511136094218, 2.0818052914231253, 2.0170825228677507, 1.9039940200491434, 1.731017362744226, 1.558296121653412, 1.4438221733118084, 1.2971888744180369, 1.1848962105350367, 1.0932162849170117, 1.0098532310596642, 1.045979580290602, 0.9573868419617885, 0.8796552407811828, 0.8149140782597045, 0.8407378071284205, 0.7792422299357804, 0.710697365168306, 0.6467959271863449, 0.8010942116833898, 0.7191139042344474, 0.6388296904045196, 0.5732121153785235, 0.44847063516438496, 0.42886656601890155, 0.4146320104757948, 0.40203589246501603, 0.31840897516456645, 0.30737981813304965, 0.30352323323450536, 0.3026253777640678 ], "peak_layer": 1, "peak_normalized": 2.2311752925780666, "population_cv": 0.5853894325437503, "rest_mean": 7.768764482063018e-05, "spike_contrast": 0.5959861363280557, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.630075927707367e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001623926218599081, 0.00015386792074423283, 0.00015152095875237137, 0.00014681021275464445, 0.00013857924204785377, 0.0001259894052054733, 0.00011341815843479708, 0.00010508635023143142, 9.441387373954058e-05, 8.62408269313164e-05, 7.956804620334879e-05, 7.350059604505077e-05, 7.61299961595796e-05, 6.96819115546532e-05, 6.402433791663498e-05, 5.931225314270705e-05, 6.119179306551814e-05, 5.67159331694711e-05, 5.17270018463023e-05, 4.7076035116333514e-05, 5.8306395658291876e-05, 5.2339586545713246e-05, 4.649622496799566e-05, 4.1720351873664185e-05, 3.2641237339703366e-05, 3.1214385671773925e-05, 3.0178345696185715e-05, 2.926155684690457e-05, 2.3174901798483916e-05, 2.237216176581569e-05, 2.2091466234996915e-05, 2.2026117221685126e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12167752534151077, "first_to_last_ratio": 5.154320181520886, "imbalance_abs_log_ratio": 1.6398352332189183, "last_quartile_mean": 0.02360690082423389, "mean": 0.06454508687602356, "normalized": [ 2.231175139109246, 2.114051177053498, 2.081805426225287, 2.0170825984642944, 1.903994030265683, 1.7310172806323025, 1.5582960978629543, 1.4438222279772381, 1.29718883510768, 1.184896287276894, 1.093216349808963, 1.0098533014799294, 1.045979536169102, 0.9573867748507232, 0.879655201241054, 0.8149140774782775, 0.8407378162482237, 0.7792421838807176, 0.7106973963780332, 0.6467959263501122, 0.8010942473612533, 0.7191138457790023, 0.638829720198085, 0.5732121198325099, 0.448470614548316, 0.42886653521122853, 0.4146320149591386, 0.4020358814661456, 0.31840895515555, 0.30737981361343175, 0.30352322421536726, 0.3026253637997583 ], "peak_layer": 1, "peak_normalized": 2.231175139109246, "population_cv": 0.5853894401227376, "rest_mean": 0.06889418133155063, "spike_contrast": 0.595986134425167, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04105997681617737, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.14401139318943024, 0.1364516168832779, 0.13437031209468842, 0.13019277155399323, 0.12289346009492874, 0.11172866076231003, 0.1005803570151329, 0.09319163113832474, 0.083727166056633, 0.0764792338013649, 0.07056174427270889, 0.06518106907606125, 0.06751284003257751, 0.06179461255669594, 0.05677742138504982, 0.05259869992733002, 0.054265495389699936, 0.05029625445604324, 0.045872025191783905, 0.041747499257326126, 0.051706697791814804, 0.046415265649557114, 0.04123331978917122, 0.03699802607297897, 0.02894657477736473, 0.027681227773427963, 0.0267624594271183, 0.025949440896511078, 0.020551733672618866, 0.019839856773614883, 0.01959093287587166, 0.019532980397343636 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.918741178196797e-06, "first_to_last_ratio": 4.256889559589457, "imbalance_abs_log_ratio": 1.448538743260589, "last_quartile_mean": 1.6253043639835596e-06, "mean": 3.7710106575161717e-06, "normalized": [ 3.111036134503483, 2.3043433850128423, 2.0000543541988076, 1.856903042177676, 1.4761204412583784, 1.3975532001518793, 1.3153833234811834, 1.2163502182025636, 1.121279589978469, 1.1184172583382423, 1.051317552929728, 0.9856458779932212, 1.025317558169754, 0.9223028019770371, 0.9122247681464483, 0.8823925941638188, 0.8042436205537018, 0.7521991935038654, 0.7015280489244919, 0.6885237102932811, 0.8268157151112371, 0.7604455810889451, 0.6890904244634548, 0.6325142134972163, 0.49413520276931655, 0.48659857961175057, 0.4811786179015675, 0.4807183548291107, 0.37702217293214657, 0.3708307648499783, 0.37628259255004715, 0.3812311064363565 ], "peak_layer": 1, "peak_normalized": 3.111036134503483, "population_cv": 0.6047224613054547, "rest_mean": 3.994058795763488e-06, "spike_contrast": 0.6425921205023355, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.566550710980664e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1731750419130549e-05, 8.689703463460319e-06, 7.542226285295328e-06, 7.002401162026217e-06, 5.566465915762819e-06, 5.2701880122185685e-06, 4.9603245315665845e-06, 4.586869636113988e-06, 4.22835728386417e-06, 4.217563400743529e-06, 3.964529696531827e-06, 3.7168811104493216e-06, 3.8664834391966e-06, 3.478013695712434e-06, 3.440009322730475e-06, 3.3275118767051026e-06, 3.0328112643474014e-06, 2.836551175278146e-06, 2.6454697490407852e-06, 2.59643024946854e-06, 3.11793087348633e-06, 2.8676483907474903e-06, 2.5985673346440308e-06, 2.385217840128462e-06, 1.8633891158970073e-06, 1.8349684296481428e-06, 1.814529696275713e-06, 1.812794039324217e-06, 1.4217546322470298e-06, 1.3984067663841415e-06, 1.4189656667440431e-06, 1.4376265653481823e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00011491361965454416, "first_to_last_ratio": 4.815696874018557, "imbalance_abs_log_ratio": 1.5718807645842947, "last_quartile_mean": 2.3862303351052105e-05, "mean": 6.221126920991082e-05, "normalized": [ 2.275296751999321, 2.0925224842292702, 2.024985525325661, 1.949051152845169, 1.8210682543299608, 1.673027934661814, 1.5224711978555032, 1.4187861804383168, 1.274895563153476, 1.1755832824885524, 1.0931627851870758, 1.0178889780531333, 1.046098089023552, 0.9645004914703617, 0.8923564167503102, 0.8324219825464898, 0.8548630177301194, 0.7953291565277032, 0.7278927794580229, 0.6660752428029298, 0.8185859571667472, 0.7386897130300317, 0.6593898256040199, 0.5965066478095707, 0.4749516886939756, 0.4558898522635054, 0.4413896066740623, 0.4292309588085209, 0.3268633002656605, 0.31649448274376557, 0.312303845893163, 0.31142685417023475 ], "peak_layer": 1, "peak_normalized": 2.275296751999321, "population_cv": 0.5648737407715131, "rest_mean": 6.615563917741249e-05, "spike_contrast": 0.6184154804352692, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.091167138540186e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014154909877106547, 0.0001301784795941785, 0.00012597691966220737, 0.00012125294597353786, 0.00011329096741974354, 0.00010408119123894721, 9.471486555412412e-05, 8.826448902254924e-05, 7.931287109386176e-05, 7.313452806556597e-05, 6.800704431952909e-05, 6.332416523946449e-05, 6.507908983621746e-05, 6.000279972795397e-05, 5.5514625273644924e-05, 5.1786028052447364e-05, 5.318211333360523e-05, 4.947843626723625e-05, 4.528313365881331e-05, 4.1437386244069785e-05, 5.092527135275304e-05, 4.595482459990308e-05, 4.102147795492783e-05, 3.710943565238267e-05, 2.9547347367042676e-05, 2.836148632923141e-05, 2.745940764725674e-05, 2.6703002731665038e-05, 2.033458076766692e-05, 1.9689523469423875e-05, 1.942881863215007e-05, 1.9374259863980114e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.519367267785128e-05, "first_to_last_ratio": 4.049851770423372, "imbalance_abs_log_ratio": 1.3986802805531915, "last_quartile_mean": 2.3505470835516462e-05, "mean": 5.353606127300736e-05, "normalized": [ 2.290218656747677, 2.0391521977718905, 1.9205616721349126, 1.8371747485589947, 1.7099486540024496, 1.5877756040230135, 1.4632374899841143, 1.3769106131996964, 1.2365773178603239, 1.1501221443012748, 1.081965026408811, 1.0124849839355623, 1.0387551076173185, 0.9627440254142965, 0.895438640452568, 0.8412318940948519, 0.8632752234551885, 0.8102372145057791, 0.7438108799024353, 0.6860735938593548, 0.8437350402761642, 0.7647523383937855, 0.6911829721367653, 0.640164753988397, 0.5390194638264668, 0.5259797411671606, 0.5155398289698411, 0.5072164058311368, 0.36377694228816476, 0.35556193943755593, 0.35248648635161195, 0.35288839910243647 ], "peak_layer": 1, "peak_normalized": 2.290218656747677, "population_cv": 0.522006447212346, "rest_mean": 5.6552214421875156e-05, "spike_contrast": 0.6586627004779664, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.724883426912129e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012260928633622825, 0.00010916817700490355, 0.00010281930735800415, 9.835509990807623e-05, 9.154391591437161e-05, 8.500325202476233e-05, 7.833597192075104e-05, 7.37143709557131e-05, 6.62014790577814e-05, 6.157300958875567e-05, 5.7924145949073136e-05, 5.420445813797414e-05, 5.5610857089050114e-05, 5.154152313480154e-05, 4.793825792148709e-05, 4.5036242227070034e-05, 4.621635525836609e-05, 4.3376909161452204e-05, 3.98207048419863e-05, 3.672967795864679e-05, 4.5170250814408064e-05, 4.0941828046925366e-05, 3.700321394717321e-05, 3.4271899494342506e-05, 2.8856979042757303e-05, 2.8158883651485667e-05, 2.759997187240515e-05, 2.715436858125031e-05, 1.9475184672046453e-05, 1.903538577607833e-05, 1.8870738131226972e-05, 1.8892254956881516e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002514238494768506, "first_to_last_ratio": 5.858443137507712, "imbalance_abs_log_ratio": 1.7678838920975262, "last_quartile_mean": 4.2916495658573695e-05, "mean": 0.00013052156418780214, "normalized": [ 2.175700208718937, 2.1468533781910737, 2.156626189980414, 2.081820074147806, 1.9605560874678556, 1.78239871858387, 1.6139249690844666, 1.4925302040410395, 1.335030130570538, 1.2140953753546508, 1.1065838540738395, 1.0144108621297776, 1.060298570529704, 0.9686444554003223, 0.8827093572640272, 0.8050838137717637, 0.8356205616435275, 0.769998030276327, 0.6982893436799542, 0.6331263204330165, 0.7830016139470163, 0.6976170559290904, 0.6139283242518799, 0.5406908798379542, 0.3991410841103365, 0.37768194897807716, 0.3609359356216191, 0.34776185509470936, 0.29820014430703956, 0.284663078565391, 0.28009631595127243, 0.2819812580627016 ], "peak_layer": 1, "peak_normalized": 2.175700208718937, "population_cv": 0.6126337690901321, "rest_mean": 0.0001400236357801111, "spike_contrast": 0.5656929071155059, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.92103775893338e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002839757944457233, 0.0002802106610033661, 0.00028148622368462384, 0.0002717224124353379, 0.00025589484721422195, 0.00023264146875590086, 0.0002106520114466548, 0.00019480737682897598, 0.00017425022087991238, 0.0001584656274644658, 0.00014443305553868413, 0.0001324024924542755, 0.00013839182793162763, 0.0001264289894606918, 0.0001152126060333103, 0.0001050807986757718, 0.00010906650277320296, 0.00010050134733319283, 9.114181739278138e-05, 8.263663767138496e-05, 0.00010219859541393816, 9.105406934395432e-05, 8.013088518055156e-05, 7.057181937852874e-05, 5.209651862969622e-05, 4.929563874611631e-05, 4.7109922888921574e-05, 4.5390421291813254e-05, 3.8921549275983125e-05, 3.715467028087005e-05, 3.6558609281200916e-05, 3.680463487398811e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "490a3ed9260ad0b92a1839e225a59c197ccf4984373e834d9df6ed56c379de39", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.01652366667985916, 0.013563322834670544, 0.012220539152622223, 0.0097816726192832, 0.008202088065445423, 0.008006753399968147, 0.008192610926926136, 0.008106541819870472, 0.008609342388808727, 0.008666405454277992, 0.009109876118600368, 0.008975221775472164, 0.009615846909582615, 0.009856111370027065, 0.009463186375796795, 0.009926749393343925, 0.010939708910882473, 0.010758070275187492, 0.011536303907632828, 0.012654256075620651, 0.014695252291858196, 0.016267452389001846, 0.01879013702273369, 0.02227928675711155, 0.021389417350292206, 0.023454682901501656, 0.024781694635748863, 0.026336129754781723, 0.026170482859015465, 0.026235800236463547, 0.030007388442754745, 0.031163375824689865 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.201577646905207e-05, "first_to_last_ratio": 4.932110786508481, "imbalance_abs_log_ratio": 1.595767047845531, "last_quartile_mean": 6.491293049748492e-06, "mean": 1.705986865374598e-05, "normalized": [ 2.1062843954634993, 2.172473088763457, 2.1330173481310855, 2.026075301568175, 1.8981950970169568, 1.7049265180158335, 1.5444094898875882, 1.4279941247542707, 1.304143449910112, 1.1896224712687855, 1.096253324873095, 1.0266908387336267, 1.0572033947314374, 0.9622174038057444, 0.8910292233541588, 0.8167935724505643, 0.8366571216551983, 0.7772132951252987, 0.7011662644209955, 0.6475304539316058, 0.7729030247398233, 0.689598266398322, 0.6131100793867855, 0.5604863427749205, 0.46076833070284307, 0.44397840145700934, 0.4298672372830245, 0.41973114466892847, 0.32654557794018146, 0.3218478363715071, 0.32040670799723164, 0.320860872417935 ], "peak_layer": 2, "peak_normalized": 2.172473088763457, "population_cv": 0.5805644210561929, "rest_mean": 1.8262358110034796e-05, "spike_contrast": 0.5785904277049709, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0566425589786377e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.5932935134042054e-05, 3.706210554810241e-05, 3.638899579527788e-05, 3.456457852735184e-05, 3.238295903429389e-05, 2.90858224616386e-05, 2.6347423045081086e-05, 2.4361392206628807e-05, 2.224851596110966e-05, 2.029480310739018e-05, 1.870193773356732e-05, 1.7515210856799968e-05, 1.8035751054412685e-05, 1.6415302525274456e-05, 1.520084151707124e-05, 1.3934391063230578e-05, 1.4273260603658855e-05, 1.3259156730782706e-05, 1.1961804375459906e-05, 1.1046784493373707e-05, 1.3185624084144365e-05, 1.1764455848606303e-05, 1.0459577424626332e-05, 9.56182338995859e-06, 7.860647201596294e-06, 7.574213213956682e-06, 7.333478606597055e-06, 7.1605581979383714e-06, 5.570824669121066e-06, 5.490681814990239e-06, 5.466096354211913e-06, 5.473844339576317e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001296065474889474, "first_to_last_ratio": 4.981819427000543, "imbalance_abs_log_ratio": 1.6057951711009326, "last_quartile_mean": 2.6015906314569293e-05, "mean": 6.850327099527931e-05, "normalized": [ 2.0631317176336217, 2.1730824382529383, 2.146215362336739, 2.044146942206416, 1.9438216945498896, 1.7429561173297294, 1.573898233305266, 1.4485558265602336, 1.3070218414216235, 1.1814367373147445, 1.0902159864460161, 1.0200822906031923, 1.0387457708358308, 0.9441162107193557, 0.8721012120223935, 0.8075112656119094, 0.832890665117298, 0.7730404866597239, 0.7008773204267549, 0.6442617083188491, 0.7641479384870525, 0.683465457573547, 0.6090232197249015, 0.557044614214288, 0.45807193966692283, 0.441967695148959, 0.42799701004285134, 0.41848476044449356, 0.3263378031557906, 0.3220971579939778, 0.3213355025096009, 0.32191707336508907 ], "peak_layer": 2, "peak_normalized": 2.1730824382529383, "population_cv": 0.5858238895510721, "rest_mean": 7.339553822357966e-05, "spike_contrast": 0.573400358946299, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.208502796245739e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014133127115201205, 0.00014886325516272336, 0.0001470227725803852, 0.00014003075193613768, 0.00013315814430825412, 0.0001193981952383183, 0.00010781717719510198, 9.92308123386465e-05, 8.953527139965445e-05, 8.093228098005056e-05, 7.46833611628972e-05, 6.987897359067574e-05, 7.115748303476721e-05, 6.467504863394424e-05, 5.974178566248156e-05, 5.53171630599536e-05, 5.7055734941968694e-05, 5.295580194797367e-05, 4.80123890156392e-05, 4.413403439684771e-05, 5.234663331066258e-05, 4.681961945607327e-05, 4.172008266323246e-05, 3.815937816398218e-05, 3.137942621833645e-05, 3.0276232791948132e-05, 2.9319195164134726e-05, 2.8667574952123687e-05, 2.235520696558524e-05, 2.2064708900870755e-05, 2.2012533008819446e-05, 2.2052372514735907e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11493638530373573, "first_to_last_ratio": 4.98181949999669, "imbalance_abs_log_ratio": 1.60579518575344, "last_quartile_mean": 0.02307116612792015, "mean": 0.060749385971575975, "normalized": [ 2.0631316304082516, 2.1730824619815525, 2.1462154581134283, 2.0441469912308308, 1.9438217938629945, 1.7429561579567567, 1.5738982612782981, 1.4485559076637762, 1.3070217763738827, 1.1814367092114706, 1.0902160293917031, 1.0200822388374744, 1.0387456719603603, 0.9441161484281956, 0.8721011745721409, 0.8075112852333144, 0.8328906721337106, 0.7730404424285328, 0.7008773139661862, 0.6442617114020169, 0.7641479159706749, 0.6834654683906575, 0.6090232364111592, 0.5570445786770215, 0.45807193597544066, 0.4419676958416594, 0.42799701081524794, 0.41848476148852237, 0.3263378087680775, 0.32209715911319503, 0.32133550584852344, 0.3219170862649435 ], "peak_layer": 2, "peak_normalized": 2.1730824619815525, "population_cv": 0.5858239032428009, "rest_mean": 0.06508789756507785, "spike_contrast": 0.5734003518757106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03732142336666584, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.1253339797258377, 0.13201342523097992, 0.1303812712430954, 0.12418067455291748, 0.11808598041534424, 0.10588351637125015, 0.09561335295438766, 0.0879988819360733, 0.07940077036619186, 0.07177155464887619, 0.06622995436191559, 0.061969369649887085, 0.06310316175222397, 0.05735447630286217, 0.05297961086034775, 0.04905581474304199, 0.050597596913576126, 0.04696173220872879, 0.04257786646485329, 0.03913850337266922, 0.04642151668667793, 0.04152010753750801, 0.03699778765439987, 0.0338401161134243, 0.027827588841319084, 0.02684926614165306, 0.026000555604696274, 0.02542269229888916, 0.01982482150197029, 0.019567204639315605, 0.01952093467116356, 0.019556265324354172 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.753675563686556e-06, "first_to_last_ratio": 4.253918229238526, "imbalance_abs_log_ratio": 1.447840494505505, "last_quartile_mean": 1.587636432986983e-06, "mean": 3.648644131715173e-06, "normalized": [ 2.903986991371279, 2.3738972852772044, 2.0636484550173555, 1.9730309898065614, 1.5024757487770555, 1.4408542771106638, 1.3294926268411962, 1.2206909715315049, 1.1367895670998007, 1.1343245437154552, 1.0524555559248938, 0.9965717010048886, 1.0284766664266607, 0.9159044401569835, 0.9130676317207874, 0.8768166573288402, 0.7991841246832521, 0.7460017873265686, 0.6928987175840244, 0.6772842439447505, 0.7872818930048806, 0.7106755394489379, 0.6442079001086913, 0.598937492646923, 0.49155975663697377, 0.488564194861926, 0.4847420251719816, 0.4867047087469123, 0.3737366741036465, 0.3778045297753075, 0.3854675643472102, 0.3924647384968829 ], "peak_layer": 1, "peak_normalized": 2.903986991371279, "population_cv": 0.6010395319776336, "rest_mean": 3.88747321688209e-06, "spike_contrast": 0.6068124306476403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3589670718138223e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.0595615094644018e-05, 8.661506399221253e-06, 7.529518825322157e-06, 7.19888794264989e-06, 5.481999323819764e-06, 5.257164502836531e-06, 4.850845471082721e-06, 4.453866949916119e-06, 4.14774058299372e-06, 4.138746589887887e-06, 3.840035788016394e-06, 3.6361354887048947e-06, 3.752545353563619e-06, 3.3418093607906485e-06, 3.3314588563371217e-06, 3.1991919513529865e-06, 2.915938466685475e-06, 2.7218950435781153e-06, 2.5281408397859195e-06, 2.4711691821721615e-06, 2.87251145891787e-06, 2.5930021365638822e-06, 2.350485374336131e-06, 2.185309767810395e-06, 1.793526621440833e-06, 1.7825968825491145e-06, 1.7686511455394793e-06, 1.775812279447564e-06, 1.3636321227750159e-06, 1.378474280500086e-06, 1.4064339666219894e-06, 1.4319641650217818e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00010877794466068735, "first_to_last_ratio": 4.650144040991601, "imbalance_abs_log_ratio": 1.5368981956768348, "last_quartile_mean": 2.3392381763187586e-05, "mean": 5.88290772043365e-05, "normalized": [ 2.086094665047828, 2.1387631343390847, 2.081886589830107, 1.9779132190019841, 1.858425229920235, 1.685830327312371, 1.5378399999356094, 1.4256522001239635, 1.285691895424893, 1.177016349317481, 1.0935933359436882, 1.0283102880092312, 1.0433458951143084, 0.9547749651290882, 0.8884378576765547, 0.8276442560837406, 0.8483650440474175, 0.7907127191539125, 0.7197037997675502, 0.663207322715193, 0.7855863227954794, 0.7052947535916103, 0.632643492598849, 0.5822024313323265, 0.48579336759194897, 0.46994818464757, 0.45555568062012447, 0.4458186318974359, 0.33465168362726294, 0.3304765071871079, 0.32922282877056575, 0.32959702144547803 ], "peak_layer": 2, "peak_normalized": 2.1387631343390847, "population_cv": 0.5615221836591825, "rest_mean": 6.276949157095428e-05, "spike_contrast": 0.598233929968243, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.755083962460048e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012272302410565317, 0.00012582146155182272, 0.0001224754669237882, 0.00011635880946414545, 0.00010932944132946432, 9.917584247887135e-05, 9.04697080841288e-05, 8.386980334762484e-05, 7.563606777694076e-05, 6.924278568476439e-05, 6.433508679037914e-05, 6.049454532330856e-05, 6.137907621450722e-05, 5.616853013634682e-05, 5.226597932050936e-05, 4.8689547838876024e-05, 4.990853267372586e-05, 4.651689960155636e-05, 4.2339510400779545e-05, 3.90158747904934e-05, 4.621531843440607e-05, 4.1491839510854334e-05, 3.7217832868918777e-05, 3.425043178140186e-05, 2.8578775527421385e-05, 2.764661803666968e-05, 2.6799920306075364e-05, 2.622709871502593e-05, 1.9687249732669443e-05, 1.944162795552984e-05, 1.936787521117367e-05, 1.938988862093538e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.041007069754414e-05, "first_to_last_ratio": 3.896952313368097, "imbalance_abs_log_ratio": 1.360194789527301, "last_quartile_mean": 2.320019939361373e-05, "mean": 5.0960711291736516e-05, "normalized": [ 2.073418805084712, 2.070705493300103, 1.9695194092323627, 1.868517934306777, 1.7463516565856263, 1.6026793418463672, 1.4766395559839594, 1.3850734214754388, 1.2521428459550328, 1.1598174296073966, 1.0827997233727307, 1.021319953269383, 1.0310088635522379, 0.9522405115352638, 0.8934796928616261, 0.8382858426322578, 0.8561291086272991, 0.804471244488103, 0.7365257997466831, 0.683193532115906, 0.8124716878605815, 0.7378124954845751, 0.6735135453221345, 0.6298294254760799, 0.554636180125419, 0.5428431825046466, 0.5332516810286245, 0.5270828016374611, 0.3733719900070218, 0.36998938084721245, 0.3703962562554523, 0.37048120787152483 ], "peak_layer": 1, "peak_normalized": 2.073418805084712, "population_cv": 0.5133586984393086, "rest_mean": 5.396500879821057e-05, "spike_contrast": 0.6437042359553637, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4737504756776615e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010566289711277932, 0.00010552462481427938, 0.0001003681099973619, 9.522100299363956e-05, 8.89953225851059e-05, 8.167367923306301e-05, 7.525060209445655e-05, 7.058432674966753e-05, 6.381009006872773e-05, 5.9105121181346476e-05, 5.5180244089569896e-05, 5.2047191275050864e-05, 5.2540945034706965e-05, 4.8526853788644075e-05, 4.5532360672950745e-05, 4.271964280633256e-05, 4.362894833320752e-05, 4.09964268328622e-05, 3.753387863980606e-05, 3.4816028346540406e-05, 4.140413511777297e-05, 3.759944956982508e-05, 3.432272933423519e-05, 3.209655551472679e-05, 2.826465424732305e-05, 2.766367470030673e-05, 2.7174884962732904e-05, 2.6860514481086284e-05, 1.902730218716897e-05, 1.885492201836314e-05, 1.887565667857416e-05, 1.88799858733546e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00023664305626880378, "first_to_last_ratio": 5.672568947923063, "imbalance_abs_log_ratio": 1.73564209236627, "last_quartile_mean": 4.171708769717952e-05, "mean": 0.00012206438873363368, "normalized": [ 2.020436105172798, 2.2182139141989996, 2.231345216253622, 2.0960079716438154, 2.010914874012047, 1.7992008227464464, 1.6294774204031262, 1.5037956431282506, 1.3510265318569439, 1.199041021272032, 1.1081855437866828, 1.0311083485666759, 1.0583798698536107, 0.9525451374397906, 0.8640397467231776, 0.7956174467156442, 0.8229808232176812, 0.7579481058232506, 0.6814030270894333, 0.6327437253152498, 0.744247194141427, 0.6591213123640995, 0.579305210409028, 0.5188112493408454, 0.40414005715569307, 0.38553326632489926, 0.3713389234486457, 0.36290767571642146, 0.30471206207452944, 0.2988035841468308, 0.30136351925091265, 0.3053046504073908 ], "peak_layer": 3, "peak_normalized": 2.231345216253622, "population_cv": 0.6172237569075844, "rest_mean": 0.00013153285544831306, "spike_contrast": 0.5392923937718314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.093466847436502e-05, "top_five_layers": [ 3, 2, 4, 1, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002466232981532812, 0.0002707649255171418, 0.00027236778987571597, 0.0002558479318395257, 0.00024546109489165246, 0.00021961834863759577, 0.00019890116527676582, 0.00018355989595875144, 0.0001649122277740389, 0.00014636020932812244, 0.00013526999100577086, 0.00012586161028593779, 0.00012919049186166376, 0.00011627183994278312, 0.00010546848352532834, 9.711655729915947e-05, 0.0001004566511255689, 9.251847222913057e-05, 8.31750439829193e-05, 7.723547605564818e-05, 9.084607881959528e-05, 8.045524009503424e-05, 7.071253639878705e-05, 6.33283780189231e-05, 4.9331109039485455e-05, 4.705988249043003e-05, 4.532725870376453e-05, 4.429810360306874e-05, 3.7194491596892476e-05, 3.647327685030177e-05, 3.67857537639793e-05, 3.726682552951388e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "40fd0d817b1a8fb04957cc5ff2887bc65937f1776b53a68242e4c7341723dda7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.01139580737799406, 0.02950318343937397, 0.04104945436120033, 0.05214058980345726, 0.01439172774553299, 0.024067245423793793, 0.030465533956885338, 0.036618731915950775, 0.016362585127353668, 0.02618422359228134, 0.033149950206279755, 0.04017872363328934, 0.025544000789523125, 0.030907167121767998, 0.03762345016002655, 0.04704070836305618, 0.022795414552092552, 0.03890290856361389, 0.04891306161880493, 0.060658931732177734, 0.033973678946495056, 0.03364448994398117, 0.04609670862555504, 0.05855504423379898, 0.015395586378872395, 0.03421308845281601, 0.058928847312927246, 0.08527950197458267, 0.006911881268024445, 0.028406694531440735, 0.05527278408408165, 0.0854114517569542 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.390311508155719e-05, "first_to_last_ratio": 5.095387404603287, "imbalance_abs_log_ratio": 1.6283357000163252, "last_quartile_mean": 6.6536874214762065e-06, "mean": 1.811846108523696e-05, "normalized": [ 2.2929525806332673, 2.1130976782069264, 2.0666904338039136, 1.9927924313254255, 1.8585936234857137, 1.6932727973648873, 1.5304627311689392, 1.421671801794904, 1.2892190508392531, 1.1855900928778078, 1.0961219453815518, 1.014146923900456, 1.060927032566774, 0.9747790234797166, 0.8967222893794864, 0.8249674532609624, 0.8460940689165917, 0.7831291514058804, 0.7104177588251133, 0.649386033089901, 0.8100343599886985, 0.7277646813689918, 0.6447855165487525, 0.5785206900712379, 0.45250607210141436, 0.4317189898164091, 0.4174322598568066, 0.40385800282682455, 0.31921174120413165, 0.30773080478647463, 0.30340519281425327, 0.30199678690853393 ], "peak_layer": 1, "peak_normalized": 2.2929525806332673, "population_cv": 0.5811766764858837, "rest_mean": 1.9317224558579727e-05, "spike_contrast": 0.6028370325080594, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1645138329186011e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.154477210249752e-05, 3.8286078051896766e-05, 3.7445250200107694e-05, 3.610633211792447e-05, 3.3674856240395457e-05, 3.067949728574604e-05, 2.7729629437089898e-05, 2.5758505216799676e-05, 2.3358665202977136e-05, 2.1481067960849032e-05, 1.9860042812069878e-05, 1.8374781575403176e-05, 1.9222365153837018e-05, 1.766149580362253e-05, 1.624722790438682e-05, 1.4947140698495787e-05, 1.5329922462115064e-05, 1.4189095054462086e-05, 1.287167651753407e-05, 1.1765875569835771e-05, 1.467657602916006e-05, 1.3185976058593951e-05, 1.1682521289912984e-05, 1.0481904610060155e-05, 8.198713658202905e-06, 7.82208371674642e-06, 7.563230155938072e-06, 7.317285508179339e-06, 5.7836255109577905e-06, 5.575608611252392e-06, 5.497235179063864e-06, 5.47171703146887e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013720781316806097, "first_to_last_ratio": 5.154303273542655, "imbalance_abs_log_ratio": 1.6398319528627574, "last_quartile_mean": 2.6620050448400434e-05, "mean": 7.278342172867269e-05, "normalized": [ 2.231162578299261, 2.1140725523598243, 2.081809525961795, 2.017090336462668, 1.9039602127395285, 1.731013692718718, 1.5582801029140219, 1.4438270127683217, 1.297193569130097, 1.1848923737412382, 1.0932281149833414, 1.009860837045549, 1.045980299568947, 0.957387934055637, 0.8796484117264644, 0.8149115280262805, 0.8407436620287138, 0.7792370696611136, 0.7106987318162484, 0.6467969744190775, 0.8010932067219945, 0.7191204217652744, 0.6388312850244199, 0.5732129901902088, 0.448470472211554, 0.42886454768570625, 0.41463001278762157, 0.40203646508675156, 0.3184117215284888, 0.30738136152216167, 0.30352664983164235, 0.30262534521733025 ], "peak_layer": 1, "peak_normalized": 2.231162578299261, "population_cv": 0.5853882312557181, "rest_mean": 7.768760001959486e-05, "spike_contrast": 0.5959877631181125, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.630085895769298e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001623916468815878, 0.00015386943414341658, 0.00015152122068684548, 0.00014681073662359267, 0.0001385767391184345, 0.0001259890996152535, 0.00011341695790179074, 0.00010508667037356645, 9.441418660571799e-05, 8.624052134109661e-05, 7.956888293847442e-05, 7.35011271899566e-05, 7.613002526341006e-05, 6.968196976231411e-05, 6.402382132364437e-05, 5.931204941589385e-05, 6.119220051914454e-05, 5.6715540267759934e-05, 5.1727085519814864e-05, 4.7076096961973235e-05, 5.83063047088217e-05, 5.234004493104294e-05, 4.649632683140226e-05, 4.1720402805367485e-05, 3.2641215511830524e-05, 3.121422923868522e-05, 3.0178191082086414e-05, 2.9261589588713832e-05, 2.317509461136069e-05, 2.2372267267201096e-05, 2.2091708160587586e-05, 2.202610812673811e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12167725898325443, "first_to_last_ratio": 5.154303204824563, "imbalance_abs_log_ratio": 1.639831939530579, "last_quartile_mean": 0.02360692690126598, "mean": 0.06454506580485031, "normalized": [ 2.231162477353291, 2.1140724141359577, 2.0818095688115155, 2.0170904137525367, 1.9039602530341841, 1.7310136901744795, 1.5582800997709945, 1.443826970315116, 1.2971936450083206, 1.1848924031019537, 1.0932281344871677, 1.009860787951356, 1.045980339365098, 0.9573879531378027, 0.8796484470446824, 0.8149115154229624, 0.8407436891757111, 0.7792370706711085, 0.7106987249961478, 0.6467969455265957, 0.8010931814132939, 0.7191204293109883, 0.6388312562189619, 0.5732129995548365, 0.44847050123207355, 0.4288645397214359, 0.41463001482239054, 0.40203647444238044, 0.3184117140434955, 0.30738135686247064, 0.3035266419790624, 0.30262534716162937 ], "peak_layer": 1, "peak_normalized": 2.231162477353291, "population_cv": 0.5853882259295132, "rest_mean": 0.06889414014639678, "spike_contrast": 0.5959877614155383, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04106006436049938, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.144010528922081, 0.13645294308662415, 0.13437053561210632, 0.13019323348999023, 0.12289123982191086, 0.11172839254140854, 0.10057929158210754, 0.09319190680980682, 0.08372744917869568, 0.07647895812988281, 0.07056248188018799, 0.06518153101205826, 0.0675128698348999, 0.061794668436050415, 0.05677696689963341, 0.05259851738810539, 0.05426585674285889, 0.050295908004045486, 0.045872095972299576, 0.041747551411390305, 0.05170661211013794, 0.046415675431489944, 0.041233405470848083, 0.036998070776462555, 0.028946558013558388, 0.02768108993768692, 0.02676232159137726, 0.025949470698833466, 0.020551905035972595, 0.019839949905872345, 0.01959114708006382, 0.01953297294676304 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.918673363998096e-06, "first_to_last_ratio": 4.2569175865719195, "imbalance_abs_log_ratio": 1.4485453271500246, "last_quartile_mean": 1.6252777328418233e-06, "mean": 3.7709897107163215e-06, "normalized": [ 3.1110512448069554, 2.304346296539455, 1.999936311003599, 1.856948328142694, 1.4760735306186028, 1.3975733840426199, 1.315376520925139, 1.216376148656068, 1.121245902760062, 1.1184741190475735, 1.0513651171862035, 0.9856602767147932, 1.025270072900951, 0.9223172106183702, 0.9122023405577108, 0.882390621923502, 0.8042742561465184, 0.7522122351984912, 0.7014935374901973, 0.6885154154574921, 0.8268174136591959, 0.7604637334111901, 0.6891177674106737, 0.632538167109275, 0.4941282098312776, 0.4865824100430069, 0.48117691929198236, 0.4807033886549141, 0.37702330245768634, 0.3708218207847471, 0.3762822105801532, 0.38124178602889996 ], "peak_layer": 1, "peak_normalized": 3.1110512448069554, "population_cv": 0.6047199362907585, "rest_mean": 3.994027649780576e-06, "spike_contrast": 0.64260567648057, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.566584839769348e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1731742233678233e-05, 8.689666174177546e-06, 7.541739250882529e-06, 7.002533038757974e-06, 5.566258096223464e-06, 5.27023485119571e-06, 4.960271326126531e-06, 4.586941940942779e-06, 4.228206762491027e-06, 4.2177543946309015e-06, 3.964687039115233e-06, 3.716914761753287e-06, 3.866282895614859e-06, 3.4780487112584524e-06, 3.439905640334473e-06, 3.327485956106102e-06, 3.032909944522544e-06, 2.836584599208436e-06, 2.645324912009528e-06, 2.5963845473597758e-06, 3.1179199595499085e-06, 2.8677009140665177e-06, 2.5986560103774536e-06, 2.385294919804437e-06, 1.8633523950484232e-06, 1.834897261687729e-06, 1.8145132116842433e-06, 1.8127275325241499e-06, 1.4217509942682227e-06, 1.3983652706883731e-06, 1.41895634442335e-06, 1.4376588524100953e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001149136905951309, "first_to_last_ratio": 4.815694891169908, "imbalance_abs_log_ratio": 1.5718803528372292, "last_quartile_mean": 2.3862327907409053e-05, "mean": 6.221135890882579e-05, "normalized": [ 2.275293471382189, 2.0925239114508343, 2.0249889212120458, 1.9490506817315036, 1.821047383587751, 1.6730348788569944, 1.5224659618516898, 1.418792087744366, 1.2749012101051704, 1.1755812366175094, 1.093166238104123, 1.0178928903694313, 1.046098218092314, 0.9644986329917807, 0.8923499840739568, 0.8324245833781815, 0.8548661709826579, 0.7953276004450885, 0.7278943614497286, 0.6660761537152818, 0.818586121883297, 0.7386933261746851, 0.6593902783338745, 0.5965063725188312, 0.4749508284557732, 0.4558878499549265, 0.44138695277868795, 0.4292311586137051, 0.3268647879844354, 0.31649601465159427, 0.3123056762318561, 0.311426054275736 ], "peak_layer": 1, "peak_normalized": 2.275293471382189, "population_cv": 0.5648730691909721, "rest_mean": 6.615572743208354e-05, "spike_contrast": 0.6184161292041498, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.091176888323389e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014154909877106547, 0.00013017875608056784, 0.00012597731256391853, 0.00012125309149269015, 0.00011328983237035573, 0.00010408177331555635, 9.471467637922615e-05, 8.8264983787667e-05, 7.931333675514907e-05, 7.313450623769313e-05, 6.80073571857065e-05, 6.332449993351474e-05, 6.507919169962406e-05, 6.0002770624123514e-05, 5.55143051315099e-05, 5.1786264521069825e-05, 5.318238618201576e-05, 4.94784108013846e-05, 4.528329736785963e-05, 4.143750265939161e-05, 5.0925355026265606e-05, 4.5955115638207644e-05, 4.10215652664192e-05, 3.710947203217074e-05, 2.9547336453106254e-05, 2.8361402655718848e-05, 2.7459282136987895e-05, 2.6703053663368337e-05, 2.033470263995696e-05, 1.9689647160703316e-05, 1.9428960513323545e-05, 1.9374238036107272e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.519206923869206e-05, "first_to_last_ratio": 4.049909152070383, "imbalance_abs_log_ratio": 1.3986944492792963, "last_quartile_mean": 2.350474187551299e-05, "mean": 5.3535959864348115e-05, "normalized": [ 2.2900191331294404, 2.039391045052068, 1.9210026615791371, 1.8368270426601154, 1.71008739314826, 1.5872578127019539, 1.4631636096399043, 1.3770182781394782, 1.236995810064802, 1.1502352236926436, 1.0819737353690493, 1.0125428958379479, 1.0387169824256042, 0.9627940963486861, 0.895516173191146, 0.8412775217154583, 0.8632420662731606, 0.8101070545557904, 0.7437235410096641, 0.6860674864525296, 0.8435748401878083, 0.7646895025830382, 0.6913289553009525, 0.640080208740369, 0.5389256211609706, 0.5259226029001551, 0.515575733832938, 0.5073004063094942, 0.36380392953064605, 0.3554700596975052, 0.3524837223636654, 0.3528848544056177 ], "peak_layer": 1, "peak_normalized": 2.2900191331294404, "population_cv": 0.5220149329094871, "rest_mean": 5.65526137911042e-05, "spike_contrast": 0.6586084384613202, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.724602865986526e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012259837239980698, 0.00010918075713561848, 0.0001028427213896066, 9.833629883360118e-05, 9.155117004411295e-05, 8.497537055518478e-05, 7.833186828065664e-05, 7.371999527094886e-05, 6.622375803999603e-05, 6.157894677016884e-05, 5.792450247099623e-05, 5.4207455832511187e-05, 5.560871068155393e-05, 5.154410609975457e-05, 4.794231790583581e-05, 4.503859963733703e-05, 4.621449261321686e-05, 4.336985875852406e-05, 3.981595364166424e-05, 3.6729281418956816e-05, 4.516158878686838e-05, 4.093838651897386e-05, 3.7010959204053506e-05, 3.4267308365087956e-05, 2.8851900424342602e-05, 2.81557713606162e-05, 2.7601841793511994e-05, 2.7158814191352576e-05, 1.9476592569844797e-05, 1.903043084894307e-05, 1.8870554413297214e-05, 1.8892029402195476e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002514511506888084, "first_to_last_ratio": 5.859513217612488, "imbalance_abs_log_ratio": 1.7680665314717174, "last_quartile_mean": 4.291331742933835e-05, "mean": 0.0001305278289009948, "normalized": [ 2.175519083700028, 2.147448905449854, 2.155906169303229, 2.0822753527857363, 1.9604833952769096, 1.7827704840526475, 1.6143096143512923, 1.49263047981926, 1.3352686262026217, 1.2139066669363794, 1.1066600661067776, 1.0143193649474604, 1.0602634006626006, 0.9683991870554761, 0.8824502085414744, 0.8051197014148838, 0.8356431104994041, 0.7697672013591956, 0.6982515926862864, 0.6332597050821357, 0.7831169912368967, 0.6976137861149029, 0.6139728290209445, 0.5405034987503755, 0.3991384270450618, 0.3775843888731109, 0.3608789237031527, 0.34760168279002573, 0.2982009104896796, 0.2846383790650737, 0.2801016579127231, 0.2819962087644011 ], "peak_layer": 1, "peak_normalized": 2.175519083700028, "population_cv": 0.6126834994698729, "rest_mean": 0.00014003051816231432, "spike_contrast": 0.5656860228000465, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.921330688986927e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002839657827280462, 0.00028030184330418706, 0.00028140575159341097, 0.00027179488097317517, 0.0002558976411819458, 0.00023270116071216762, 0.00021071232913527638, 0.00019482981588225812, 0.00017428971477784216, 0.0001584486017236486, 0.00014444993576034904, 0.0001323969045188278, 0.00013839387975167483, 0.00012640304339583963, 0.00011518430983414873, 0.00010509052663110197, 0.0001090746809495613, 0.0001004760415526107, 9.11412644200027e-05, 8.265801443485543e-05, 0.00010221856064163148, 9.105801291298121e-05, 8.014054037630558e-05, 7.055074820527807e-05, 5.209867231315002e-05, 4.92852705065161e-05, 4.7104742407100275e-05, 4.5371692976914346e-05, 3.892351742251776e-05, 3.715322964126244e-05, 3.6561061278916895e-05, 3.6808352888328955e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "79ad652a3aa7b0850b826f149977b0495d569b726bb8d66c66c52638215ffd53", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.021054552868008614, 0.038058556616306305, 0.04883526638150215, 0.05791458487510681, 0.016990283504128456, 0.027475843206048012, 0.03445403650403023, 0.040619369596242905, 0.019055280834436417, 0.029903952032327652, 0.037492990493774414, 0.04475866258144379, 0.0285621490329504, 0.03507128730416298, 0.04228900745511055, 0.05219251662492752, 0.027010982856154442, 0.04389137029647827, 0.054182786494493484, 0.06662839651107788, 0.03837457299232483, 0.041143495589494705, 0.05716610327363014, 0.07329611480236053, 0.028997479006648064, 0.053451184183359146, 0.08026835322380066, 0.10854664444923401, 0.027061885222792625, 0.05247335880994797, 0.08323372155427933, 0.11460508406162262 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.201575941602641e-05, "first_to_last_ratio": 4.932102112877248, "imbalance_abs_log_ratio": 1.5957652892396856, "last_quartile_mean": 6.491301007827133e-06, "mean": 1.705989417644105e-05, "normalized": [ 2.106292333194075, 2.172491163348758, 2.1330241796302363, 2.0259918761437463, 1.8982135819397121, 1.704913518211797, 1.5444411923191312, 1.4279770610625449, 1.3041442710446836, 1.1896258094559513, 1.0962581888548657, 1.0266942074295144, 1.0572238841977288, 0.9622016766881734, 0.891040738475761, 0.8167872858482652, 0.8366683449379246, 0.7772035491548255, 0.7011655353024898, 0.6475278325168645, 0.7728997359529347, 0.6896078437753965, 0.6131124674701751, 0.5604884364037679, 0.4607666284385362, 0.44397715080639827, 0.4298684867451416, 0.41972950379888196, 0.3265460490205608, 0.3218480479207054, 0.3204072149176842, 0.3208622049927692 ], "peak_layer": 2, "peak_normalized": 2.172491163348758, "population_cv": 0.5805628153884638, "rest_mean": 1.8262379702112714e-05, "spike_contrast": 0.5785923034220807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0566472337814047e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.593312430894002e-05, 3.706246934598312e-05, 3.638916678028181e-05, 3.456320700934157e-05, 3.23833228321746e-05, 2.9085644200677052e-05, 2.6348003302700818e-05, 2.436113754811231e-05, 2.2248563254834153e-05, 2.029489041888155e-05, 1.8702048691920936e-05, 1.751529453031253e-05, 1.803612758521922e-05, 1.641505878069438e-05, 1.5201060705294367e-05, 1.393430466123391e-05, 1.427347342541907e-05, 1.3259010302135721e-05, 1.1961809832428116e-05, 1.1046756299037952e-05, 1.3185587704356294e-05, 1.1764636838051956e-05, 1.0459633813297842e-05, 9.561873412167188e-06, 7.86062992119696e-06, 7.5742032095149625e-06, 7.333510893658968e-06, 7.160540917539038e-06, 5.570841040025698e-06, 5.490693638421362e-06, 5.466113179863896e-06, 5.473875262396177e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.000129606451082509, "first_to_last_ratio": 4.9818074922726145, "imbalance_abs_log_ratio": 1.6057927754415957, "last_quartile_mean": 2.601594928819395e-05, "mean": 6.850338621688934e-05, "normalized": [ 2.0631371693799077, 2.173061151786652, 2.146221311616304, 2.04412417319785, 1.9437850741564366, 1.7429527608527258, 1.5739287245233284, 1.448561249882341, 1.3070356812110293, 1.181442503713818, 1.0902177639657034, 1.0200948073975256, 1.038760061861336, 0.9441053821903842, 0.8720979926456799, 0.8075042780959905, 0.832896699126015, 0.7730365841967101, 0.7008771505867426, 0.6442638641806842, 0.7641491492103867, 0.6834677068149604, 0.6090231512754765, 0.5570467574542299, 0.45807302792658366, 0.4419659692950432, 0.4279961308389998, 0.4184841096663127, 0.3263401485679923, 0.32209706763737483, 0.3213341654300619, 0.32191823131541414 ], "peak_layer": 2, "peak_normalized": 2.173061151786652, "population_cv": 0.5858213540607353, "rest_mean": 7.33956448702547e-05, "spike_contrast": 0.5734017265344913, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2085189488716426e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014133188233245164, 0.0001488620473537594, 0.00014702342741657048, 0.0001400294277118519, 0.00013315585965756327, 0.00011939816613448784, 0.0001078194472938776, 9.923135075950995e-05, 8.953637006925419e-05, 8.09328121249564e-05, 7.468360854545608e-05, 6.987994856899604e-05, 7.115858170436695e-05, 6.467441562563181e-05, 5.974166560918093e-05, 5.531677743420005e-05, 5.705624425900169e-05, 5.295562368701212e-05, 4.8012458137236536e-05, 4.413425631355494e-05, 5.2346804295666516e-05, 4.681985228671692e-05, 4.172014814685099e-05, 3.8159589166752994e-05, 3.1379553547594696e-05, 3.02761654893402e-05, 2.9319184250198305e-05, 2.8667578590102494e-05, 2.2355405235430226e-05, 2.2064739823690616e-05, 2.201247843913734e-05, 2.2052488930057734e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11493630055338144, "first_to_last_ratio": 4.9818077321878835, "imbalance_abs_log_ratio": 1.6057928235998724, "last_quartile_mean": 0.02307120361365378, "mean": 0.06074948905734345, "normalized": [ 2.063137205162244, 2.1730611136953453, 2.1462213824598626, 2.0441242673588023, 1.9437851361343719, 1.7429527097582802, 1.5739287045014374, 1.4485612988475673, 1.3070356248971926, 1.1814425536696513, 1.0902177360905216, 1.020094857247625, 1.0387600983655463, 0.9441054093552858, 0.8720980390057368, 0.8075042733286111, 0.8328966787842622, 0.7730365551257176, 0.7008771058026076, 0.6442638069074199, 0.7641491334974229, 0.6834676813377316, 0.6090231227925368, 0.557046760858952, 0.4580729983384807, 0.4419659647123029, 0.42799613124198876, 0.418484082023093, 0.32634013714750504, 0.3220970724629912, 0.32133413272526656, 0.3219182263636403 ], "peak_layer": 2, "peak_normalized": 2.1730611136953453, "population_cv": 0.5858213710563731, "rest_mean": 0.06508799324984904, "spike_contrast": 0.5734017067410487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0373215664178133, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.12533453106880188, 0.13201235234737396, 0.13038185238838196, 0.12417950481176376, 0.11808395385742188, 0.10588348656892776, 0.09561536461114883, 0.0879993587732315, 0.07940174639225006, 0.0717720314860344, 0.0662301704287529, 0.061970241367816925, 0.06310414522886276, 0.0573539212346077, 0.05297951027750969, 0.049055472016334534, 0.05059804767370224, 0.046961575746536255, 0.04257792606949806, 0.03913869708776474, 0.04642166942358017, 0.041520312428474426, 0.03699784353375435, 0.03384030610322952, 0.027827700600028038, 0.026849206537008286, 0.026000546291470528, 0.02542269416153431, 0.01982499659061432, 0.019567232578992844, 0.01952088437974453, 0.01955636776983738 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.7535999619394715e-06, "first_to_last_ratio": 4.253913636706542, "imbalance_abs_log_ratio": 1.4478394149044844, "last_quartile_mean": 1.5876203747211548e-06, "mean": 3.648633160935333e-06, "normalized": [ 2.9040847125163616, 2.373904173886903, 2.063563551858565, 1.9729206379098345, 1.502439510808684, 1.4407906834302517, 1.3294895201906802, 1.2207633158069082, 1.136833740861507, 1.1342758569990883, 1.0524618363430551, 0.9966191922064839, 1.0284603781292727, 0.9159325573360332, 0.9130731814372068, 0.8768378643711425, 0.7992107068730414, 0.7459864568856505, 0.6929026705299385, 0.6772775559667719, 0.7873543673999568, 0.7106391017928749, 0.6441870912399603, 0.5989718855930185, 0.49154718207331655, 0.4885345051390287, 0.4847150036087574, 0.4867019034315765, 0.3737492954398724, 0.37780977871978977, 0.38548196584593536, 0.3924798153685328 ], "peak_layer": 1, "peak_normalized": 2.9040847125163616, "population_cv": 0.6010372604548386, "rest_mean": 3.887456521759439e-06, "spike_contrast": 0.6068201661629119, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3589870124851585e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.059593978425255e-05, 8.661505489726551e-06, 7.529186405008659e-06, 7.198463663371513e-06, 5.481850621436024e-06, 5.256916665530298e-06, 4.85081955048372e-06, 4.454117515706457e-06, 4.14788928537746e-06, 4.138556505495217e-06, 3.840047156700166e-06, 3.636297833509161e-06, 3.752474640350556e-06, 3.341901901876554e-06, 3.3314690881525166e-06, 3.1992597087082686e-06, 2.916026687671547e-06, 2.7218309242016403e-06, 2.528147660996183e-06, 2.4711373498575995e-06, 2.872767254302744e-06, 2.592861392258783e-06, 2.350402382944594e-06, 2.185428684242652e-06, 1.7934753486770205e-06, 1.782483195711393e-06, 1.7685472357698018e-06, 1.775796704350796e-06, 1.3636740732181352e-06, 1.3784892871626653e-06, 1.406482283528021e-06, 1.4320148693514057e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001087780183297582, "first_to_last_ratio": 4.650142263541028, "imbalance_abs_log_ratio": 1.536897813441167, "last_quartile_mean": 2.339240654691821e-05, "mean": 5.882917196231574e-05, "normalized": [ 2.086098725675131, 2.138759442007871, 2.081889667800162, 1.9778974178151685, 1.8584148157319034, 1.6858251383046516, 1.537848282997998, 1.4256580666230054, 1.285692298109234, 1.1770210084696262, 1.0935923165353414, 1.0283135788544562, 1.0433544799560879, 0.9547703352579274, 0.888436859519986, 0.8276396454668482, 0.848368006338972, 0.7907124968033461, 0.7197062272212832, 0.6632080478180333, 0.7855866652595299, 0.7052955964220388, 0.6326444524509259, 0.5822042763475935, 0.48579283246827537, 0.46994724216827105, 0.45555494684240355, 0.44581791380348257, 0.3346526596662172, 0.330475418320936, 0.3292227313590583, 0.32959840758423514 ], "peak_layer": 2, "peak_normalized": 2.138759442007871, "population_cv": 0.5615212363220359, "rest_mean": 6.27695851478536e-05, "spike_contrast": 0.5982346493442672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.755094076041132e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012272346066311002, 0.0001258214469999075, 0.00012247584527358413, 0.0001163580673164688, 0.00010932900477200747, 9.917569695971906e-05, 9.047034109244123e-05, 8.387028356082737e-05, 7.563621329609305e-05, 6.924317131051794e-05, 6.433513044612482e-05, 6.0494836361613125e-05, 6.13796801189892e-05, 5.616834823740646e-05, 5.226600478636101e-05, 4.868935502599925e-05, 4.9908787332242355e-05, 4.651696144719608e-05, 4.233972140355036e-05, 3.9015980291878805e-05, 4.6215413021855056e-05, 4.149195592617616e-05, 3.7217949284240603e-05, 3.425059549044818e-05, 2.8578790079336613e-05, 2.7646607122733258e-05, 2.6799920306075364e-05, 2.622709871502593e-05, 1.9687338863150217e-05, 1.9441595213720575e-05, 1.9367900677025318e-05, 1.93900013982784e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.041136763698887e-05, "first_to_last_ratio": 3.896756618520285, "imbalance_abs_log_ratio": 1.3601445708574575, "last_quartile_mean": 2.320169733138755e-05, "mean": 5.096003008020489e-05, "normalized": [ 2.073194376167996, 2.071105252365612, 1.96987669569892, 1.868253501375736, 1.7461955294439868, 1.6029130762574872, 1.4764811084477982, 1.3852794036995026, 1.2521995618218273, 1.1598580624238308, 1.0825747595090334, 1.0211644142159193, 1.0309911631402593, 0.9519908151976145, 0.8935121251146112, 0.8382787729230907, 0.8561012891122731, 0.8043820538962967, 0.7366169572228874, 0.6831815336454671, 0.8123345595077685, 0.737879683501474, 0.6735151955051606, 0.6297835892231947, 0.5547719514435682, 0.5428008237338267, 0.5332896849902665, 0.5271935397804924, 0.37339953996344866, 0.3699162629784263, 0.37044439781611194, 0.3705203198761118 ], "peak_layer": 1, "peak_normalized": 2.073194376167996, "population_cv": 0.5133726204518009, "rest_mean": 5.3964246574280505e-05, "spike_contrast": 0.64370881124749, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.473726101219654e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010565004777163267, 0.00010554358595982194, 0.00010038497566711158, 9.520625462755561e-05, 8.898617670638487e-05, 8.16844985820353e-05, 7.524152169935405e-05, 7.059388008201495e-05, 6.38121273368597e-05, 5.910640174988657e-05, 5.516804230865091e-05, 5.2038569265278056e-05, 5.253934068605304e-05, 4.8513480578549206e-05, 4.553340477286838e-05, 4.271871148375794e-05, 4.362694744486362e-05, 4.099133366253227e-05, 3.753802229766734e-05, 3.481495150481351e-05, 4.1396593587705866e-05, 3.760237086680718e-05, 3.432235462241806e-05, 3.20937906508334e-05, 2.8271195333218202e-05, 2.76611463050358e-05, 2.717645838856697e-05, 2.6865798645303585e-05, 1.902845178847201e-05, 1.8850943888537586e-05, 1.887785765575245e-05, 1.8881726646213792e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00023663779211346991, "first_to_last_ratio": 5.672677986629943, "imbalance_abs_log_ratio": 1.7356613142849313, "last_quartile_mean": 4.171535783825675e-05, "mean": 0.00012206177893858694, "normalized": [ 2.0204564142034975, 2.2187758849519086, 2.231495928529079, 2.0959130632005167, 2.010595686112895, 1.799588002892838, 1.6290705588593046, 1.5034830183184902, 1.3508380843403627, 1.1990338730444932, 1.108139972391173, 1.0309048348839545, 1.0584610348587842, 0.9524995764002239, 0.8641330297603085, 0.795578425477979, 0.8231670526464523, 0.757893675020928, 0.6814420357365967, 0.6326769608985996, 0.7444868783224939, 0.6591411274404338, 0.5794466495741255, 0.5187294118731414, 0.40422067568857084, 0.3855350120263851, 0.37131717783399165, 0.36287239745994854, 0.304694852796421, 0.298784221840333, 0.30136301825835976, 0.30526146435741114 ], "peak_layer": 3, "peak_normalized": 2.231495928529079, "population_cv": 0.6172354521995556, "rest_mean": 0.00013152823615840775, "spike_contrast": 0.5393739931714252, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.094290995155462e-05, "top_five_layers": [ 3, 2, 4, 1, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00024662050418555737, 0.00027082773158326745, 0.0002723803627304733, 0.00025583087699487805, 0.0002454168861731887, 0.00021966091298963875, 0.00019884725043084472, 0.000183517811819911, 0.0001648856996325776, 0.00014635620755143464, 0.00013526153634302318, 0.00012583407806232572, 0.0001291976368520409, 0.00011626379273366183, 0.00010547761485213414, 9.710971789900213e-05, 0.00010047723480965942, 9.250985021935776e-05, 8.317802712554112e-05, 7.722567534074187e-05, 9.087339276447892e-05, 8.04559385869652e-05, 7.072828884702176e-05, 6.331703480100259e-05, 4.933989475830458e-05, 4.705908941105008e-05, 4.5323635276872665e-05, 4.42928503616713e-05, 3.719159576576203e-05, 3.6470133636612445e-05, 3.678490611491725e-05, 3.726075738086365e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "0db2d20deb5c148ef675d303417844641f9632a980cb7b1bc2b39e8bf6c514f2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04405451565980911, 0.0240375604480505, 0.030349314212799072, 0.034919846802949905, 0.038470037281513214, 0.02659580484032631, 0.028048591688275337, 0.029997555539011955, 0.031361404806375504, 0.0246393084526062, 0.025546060875058174, 0.02669469080865383, 0.027990806847810745, 0.02195541374385357, 0.023044798523187637, 0.02394026517868042, 0.025360919535160065, 0.02148214913904667, 0.02322523109614849, 0.023834573104977608, 0.02502058818936348, 0.01980534754693508, 0.02057415433228016, 0.02201046608388424, 0.02379545010626316, 0.022529911249876022, 0.023956971243023872, 0.02642984129488468, 0.030454812571406364, 0.02776513062417507, 0.030121317133307457, 0.03041756898164749 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9001171494892333e-05, "first_to_last_ratio": 4.396109463475654, "imbalance_abs_log_ratio": 1.480719936931006, "last_quartile_mean": 4.322269873568985e-06, "mean": 1.3701130082921509e-05, "normalized": [ 1.0077312064371016, 2.0216860298126127, 1.9115786719811696, 1.5699942654414412, 1.2912410675604775, 1.3448969617811173, 1.0829684927533307, 0.864562725693079, 1.472694590311165, 1.8785554526189956, 1.4195158427110435, 1.1323106177566131, 1.5536879058091357, 1.3271158818759365, 1.4866965999456043, 1.0117258821924624, 0.6982931262632973, 0.8871860339483304, 1.0814660916038894, 0.9542868684601444, 1.3260318801629343, 0.8711143311473454, 0.8200899788684414, 0.46082446220796464, 0.6310998230199332, 0.5173662703583513, 0.4802648347536266, 0.33073499649835436, 0.17451899850392552, 0.1289952247275187, 0.17263606827795114, 0.08812881651670629 ], "peak_layer": 2, "peak_normalized": 2.0216860298126127, "population_cv": 0.5264799713353324, "rest_mean": 1.4153185942531207e-05, "spike_contrast": 0.7955825979217899, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1260028441029134e-05, "top_five_layers": [ 2, 3, 10, 4, 13 ], "top_five_spike_overlap": 0 }, "values": [ 1.3807056348014157e-05, 2.7699383281287737e-05, 2.619078804855235e-05, 2.1510695660253987e-05, 1.7691461835056543e-05, 1.8426608221489005e-05, 1.4837892194918822e-05, 1.184548636956606e-05, 2.017758015426807e-05, 2.5738332624314353e-05, 1.9448971215751953e-05, 1.551393506815657e-05, 2.128728010575287e-05, 1.81829873326933e-05, 2.0369423509691842e-05, 1.3861787920177449e-05, 9.56740495894337e-06, 1.2155451258877292e-05, 1.4817307601333596e-05, 1.3074808521196246e-05, 1.816813528421335e-05, 1.1935250768146943e-05, 1.1236159480176866e-05, 6.31381590210367e-06, 8.646780770504847e-06, 7.0885025706957094e-06, 6.580170975212241e-06, 4.5314432099985424e-06, 2.391107500443468e-06, 1.7673803540674271e-06, 2.365309228480328e-06, 1.2074643791493145e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.191661304910667e-05, "first_to_last_ratio": 3.797990683695372, "imbalance_abs_log_ratio": 1.3344721594369466, "last_quartile_mean": 1.6302465752460193e-05, "mean": 4.225009619318598e-05, "normalized": [ 0.9174753286219919, 2.2626595758448156, 1.7722563272438279, 1.463236175471811, 1.3866667550506941, 1.6673315743394923, 1.2737488857960582, 0.9804544105398723, 1.1727460803814453, 1.3858997244842928, 1.1458588519796304, 1.012187071256809, 1.1964358555891934, 1.2230862069043256, 1.1426917941369261, 1.0392305817059992, 0.812301299394157, 0.8125846736189017, 1.1589483129789653, 1.1474138368245612, 1.578094611394048, 0.9268792886892305, 0.8930796579738131, 0.5418827272180105, 0.8135175439777245, 0.634708408163733, 0.5701529871597378, 0.4148669257029605, 0.21665436196405244, 0.1431611577547963, 0.20865076223123533, 0.0851382456068877 ], "peak_layer": 2, "peak_normalized": 2.2626595758448156, "population_cv": 0.48660988012988077, "rest_mean": 4.2635896172079675e-05, "spike_contrast": 0.9420882381607695, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.0166776307160036e-05, "top_five_layers": [ 2, 3, 6, 21, 4 ], "top_five_spike_overlap": 1 }, "values": [ 3.8763420889154077e-05, 9.559758473187685e-05, 7.487800030503422e-05, 6.182186916703358e-05, 5.858680378878489e-05, 7.044491940177977e-05, 5.3816012950846925e-05, 4.142429315834306e-05, 4.954863470629789e-05, 5.855439667357132e-05, 4.8412646719953045e-05, 4.276500112609938e-05, 5.054952998762019e-05, 5.1675509894266725e-05, 4.8278838221449405e-05, 4.3907592043979093e-05, 3.43198080372531e-05, 3.4331780625507236e-05, 4.89656777062919e-05, 4.8478344979230314e-05, 6.667464913334697e-05, 3.916073910659179e-05, 3.773270145757124e-05, 2.2894597350386903e-05, 3.437119448790327e-05, 2.6816491299541667e-05, 2.4089018552331254e-05, 1.7528167518321425e-05, 9.15366763365455e-06, 6.048572686268017e-06, 8.81551477505127e-06, 3.597099066610099e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.054908270947635174, "first_to_last_ratio": 3.797990637672001, "imbalance_abs_log_ratio": 1.3344721473191257, "last_quartile_mean": 0.01445718965260312, "mean": 0.03746780757501256, "normalized": [ 0.9174753670275252, 2.2626595668392633, 1.7722562063201177, 1.4632360819755696, 1.3866667949681875, 1.6673315955377577, 1.2737488958767915, 0.9804544532014017, 1.1727461605745655, 1.3858997200707546, 1.1458588680193664, 1.0121870939065154, 1.1964358991827213, 1.2230861591318958, 1.1426918379859348, 1.0392305853812045, 0.8123012456223383, 0.8125847103705666, 1.158948357357931, 1.1474137979378307, 1.5780945861023632, 0.926879317506979, 0.8930796013491593, 0.5418826865229676, 0.8135175788378592, 0.6347083934759172, 0.5701530007014057, 0.41486691525311564, 0.21665436405077207, 0.1431611567511669, 0.2086507602497586, 0.08513824191029651 ], "peak_layer": 2, "peak_normalized": 2.2626595668392633, "population_cv": 0.4866098746823269, "rest_mean": 0.03780993893397627, "spike_contrast": 0.9420882244430143, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.035620298236608505, "top_five_layers": [ 2, 3, 6, 21, 4 ], "top_five_spike_overlap": 1 }, "values": [ 0.034375790506601334, 0.08477689325809479, 0.06640255451202393, 0.05482424795627594, 0.051955364644527435, 0.062471259385347366, 0.04772457852959633, 0.03673547878861427, 0.0439402274787426, 0.0519266240298748, 0.042932819575071335, 0.03792443126440048, 0.04482783004641533, 0.04582635685801506, 0.04281415790319443, 0.03893769159913063, 0.030435146763920784, 0.0304457675665617, 0.043423254042863846, 0.04299107939004898, 0.059127744287252426, 0.03472813591361046, 0.033461734652519226, 0.020303156226873398, 0.030480720102787018, 0.023781131953001022, 0.021362382918596268, 0.01554415374994278, 0.008117564022541046, 0.005363934673368931, 0.007817686535418034, 0.003189943265169859 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.201552186103072e-05, "first_to_last_ratio": 5.505409601247022, "imbalance_abs_log_ratio": 1.705731172719516, "last_quartile_mean": 2.182493716418321e-06, "mean": 8.921801814665287e-06, "normalized": [ 1.8522267980560887, 1.3991189499642396, 1.5189623710595221, 1.7095356122655745, 1.0880389595429378, 0.9409894011627431, 1.2244978988385578, 1.0407064664287273, 0.8019872963111272, 1.6142561275113876, 1.5038986961003484, 1.2795093808294922, 1.518204136149992, 1.3587318804660677, 1.8279910088209055, 1.9335497857688762, 1.3708485517952869, 1.171301878733052, 0.9049971990240303, 0.9405561531974967, 0.9612715223781346, 0.8560463973337525, 0.6765022731234531, 0.5492730955683951, 0.30272551699640965, 0.3421729481134854, 0.33897188277967555, 0.26236357588022213, 0.18272919759715678, 0.18185216354703881, 0.19745928436567797, 0.14872359029014504 ], "peak_layer": 16, "peak_normalized": 1.9335497857688762, "population_cv": 0.5460082571286404, "rest_mean": 9.468404658508479e-06, "spike_contrast": 0.6305335136418331, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.970146457912051e-06, "top_five_layers": [ 16, 1, 15, 4, 10 ], "top_five_spike_overlap": 0 }, "values": [ 1.6525200408068486e-05, 1.2482661986723542e-05, 1.355188123852713e-05, 1.5252137927745935e-05, 9.707267963676713e-06, 8.395320946874563e-06, 1.0924727575911675e-05, 9.284976840717718e-06, 7.155171715567121e-06, 1.4402073247765657e-05, 1.3417486115940847e-05, 1.1415529115765821e-05, 1.3545116416935343e-05, 1.212233655678574e-05, 1.6308973499690183e-05, 1.7250747987418436e-05, 1.223043909703847e-05, 1.0450123227201402e-05, 8.074205652519595e-06, 8.391455594392028e-06, 8.576274012739304e-06, 7.637476301169954e-06, 6.035619207978016e-06, 4.900505700788926e-06, 2.700857066884055e-06, 3.052799229408265e-06, 3.024239958904218e-06, 2.3407558273902396e-06, 1.630273686714645e-06, 1.6224489627347793e-06, 1.7616926015762147e-06, 1.3268823977341526e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.77098144529009e-05, "first_to_last_ratio": 5.786351228183689, "imbalance_abs_log_ratio": 1.7555019078197227, "last_quartile_mean": 8.245233061643376e-06, "mean": 3.073857808999492e-05, "normalized": [ 0.8362577566227221, 2.373745168443029, 1.8117198501099894, 1.5751424117641866, 1.5278011706530124, 1.815487592878854, 1.4046395949024106, 1.0721280996591929, 1.2352562716436093, 1.4523847721238452, 1.2122549920407257, 1.0201321842789222, 1.2203169082313816, 1.2783429873250631, 1.0525616380439955, 1.010813722620164, 0.8760394275203864, 0.9273309105161657, 1.2093309822135678, 1.0693992526347946, 1.428708411485824, 0.9918157658703739, 0.9182234708569954, 0.534268291669333, 0.5596528890045244, 0.41161184317293326, 0.4078398397243235, 0.27957479810482855, 0.14962079640520384, 0.11736499492778929, 0.14511937346068207, 0.07511383109117112 ], "peak_layer": 2, "peak_normalized": 2.373745168443029, "population_cv": 0.5427210092082145, "rest_mean": 3.138446525688913e-05, "spike_contrast": 0.8682890457336828, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7250787388766186e-05, "top_five_layers": [ 2, 6, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.5705374355311505e-05, 7.296555122593418e-05, 5.568969208979979e-05, 4.8417638026876375e-05, 4.696243559010327e-05, 5.5805507145123556e-05, 4.317662387620658e-05, 3.295569331385195e-05, 3.797002136707306e-05, 4.464424273464829e-05, 3.726299473783001e-05, 3.1357412808574736e-05, 3.7510806578211486e-05, 3.929444574168883e-05, 3.235424810554832e-05, 3.107097654719837e-05, 2.692820635274984e-05, 2.8504833608167246e-05, 3.7173114833422005e-05, 3.287181243649684e-05, 4.391646507428959e-05, 3.0487006370094605e-05, 2.8224883863003924e-05, 1.6422647604485974e-05, 1.720293403195683e-05, 1.265236278413795e-05, 1.253641676157713e-05, 8.593731763539836e-06, 4.599130534188589e-06, 3.6076330616197083e-06, 4.460763193492312e-06, 2.3088923626346514e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.6240980534785194e-05, "first_to_last_ratio": 8.81501945263024, "imbalance_abs_log_ratio": 2.176457022612032, "last_quartile_mean": 4.111276297180666e-06, "mean": 2.2519588263492096e-05, "normalized": [ 0.7284164778364034, 2.389714365593298, 1.8344675527063727, 1.6124517982477564, 1.660867848829399, 1.9609031976200701, 1.5230683202508775, 1.1645840254645428, 1.3073819892255008, 1.4901651727836733, 1.2625908178604672, 1.0494073392406849, 1.21297826715993, 1.378958269273871, 0.9917742879119321, 0.9095282929708826, 0.9198153022966056, 1.0487426528518113, 1.2584126391221928, 0.9987518391352217, 1.3085415757850667, 1.0724225800006402, 0.9485461675021898, 0.5069936019893233, 0.33739074000009295, 0.273271867252871, 0.28263222104028385, 0.19007385384653275, 0.10699675859843288, 0.10147279993418688, 0.10196626637254216, 0.0667111112963436 ], "peak_layer": 2, "peak_normalized": 2.389714365593298, "population_cv": 0.5911439983506431, "rest_mean": 2.320860870794503e-05, "spike_contrast": 0.8099958985051443, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.879887786344625e-05, "top_five_layers": [ 2, 6, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.640363916521892e-05, 5.38153835805133e-05, 4.13114539696835e-05, 3.63117505912669e-05, 3.74020601157099e-05, 4.4158732634969056e-05, 3.4298871469218284e-05, 2.6225952751701698e-05, 2.944170410046354e-05, 3.355790613568388e-05, 2.8433025363483466e-05, 2.3632221200386994e-05, 2.731577114900574e-05, 3.105357245658524e-05, 2.233434861409478e-05, 2.048220267170109e-05, 2.0713861886179075e-05, 2.3617252736585215e-05, 2.833893449860625e-05, 2.2491480194730684e-05, 2.9467817512340844e-05, 2.4150514946086332e-05, 2.136086914106272e-05, 1.1417287169024348e-05, 7.597900548717007e-06, 6.153969934530323e-06, 6.3647612478234805e-06, 4.280384928279091e-06, 2.409522949164966e-06, 2.2851256744615966e-06, 2.296238335475209e-06, 1.5023067589936545e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011762118538172217, "first_to_last_ratio": 4.228719768518167, "imbalance_abs_log_ratio": 1.4418992920522087, "last_quartile_mean": 2.78148451116067e-05, "mean": 8.002406778473414e-05, "normalized": [ 0.9210338573301938, 2.3087354790458674, 1.72875802725484, 1.5227237833085348, 1.3615568761930628, 1.6698981403826305, 1.262630340241389, 0.9832444969618955, 1.1432437822572334, 1.444183076942276, 1.2101593686516263, 1.0400906407977921, 1.2308343305675216, 1.1478679901804696, 1.125011533923313, 1.162732392396995, 0.8650388354268836, 0.807876464469789, 1.2254554690773087, 1.1623670673328852, 1.5536447585331754, 0.8757401866950771, 0.9111182549928211, 0.5554068863324947, 0.8132826025949527, 0.4896112785691883, 0.5227427522902547, 0.38035098946634355, 0.18821446379390167, 0.12866317105200828, 0.18038891087911704, 0.07739379205815819 ], "peak_layer": 2, "peak_normalized": 2.3087354790458674, "population_cv": 0.5019050745008682, "rest_mean": 8.088597830161841e-05, "spike_contrast": 0.9318024282590781, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.536975099355913e-05, "top_five_layers": [ 2, 3, 6, 21, 4 ], "top_five_spike_overlap": 1 }, "values": [ 7.370487583102658e-05, 0.00018475440447218716, 0.00013834224955644459, 0.000121854551252909, 0.00010895731975324452, 0.00013363204197958112, 0.00010104081593453884, 7.868322427384555e-05, 9.14870179258287e-05, 0.00011556940444279462, 9.684187534730881e-05, 8.323228394147009e-05, 9.849636990111321e-05, 9.185706585412845e-05, 9.002799924928695e-05, 9.304657578468323e-05, 6.922392640262842e-05, 6.464956095442176e-05, 9.806593152461573e-05, 9.301734098698944e-05, 0.00012432897347025573, 7.008029206190258e-05, 7.29113889974542e-05, 4.444591831997968e-05, 6.508218211820349e-05, 3.9180686144391075e-05, 4.183200144325383e-05, 3.043723336304538e-05, 1.5061687008710578e-05, 1.0296150321664754e-05, 1.4435454431804828e-05, 6.19336606177967e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "f2daf0e86334d46ef8de0e95de81ea9f5fd3989918e22ce51c09039c50c57cf2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.018189694732427597, 0.02713710628449917, 0.03182194381952286, 0.03608781099319458, 0.026185201480984688, 0.02762730047106743, 0.028690988197922707, 0.03034786321222782, 0.02331089787185192, 0.024178674444556236, 0.024928271770477295, 0.0256717037409544, 0.020905915647745132, 0.021578654646873474, 0.022400982677936554, 0.023724498227238655, 0.020535636693239212, 0.021988606080412865, 0.023164646700024605, 0.024238524958491325, 0.019457563757896423, 0.019637931138277054, 0.02042427472770214, 0.021937908604741096, 0.021518733352422714, 0.02267260104417801, 0.024102909490466118, 0.025153208523988724, 0.023361310362815857, 0.025513608008623123, 0.027568820863962173, 0.028944427147507668 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.237074980053876e-05, "first_to_last_ratio": 1.549599178512509, "imbalance_abs_log_ratio": 0.43799630298177145, "last_quartile_mean": 7.983193313521042e-06, "mean": 1.1284510890163801e-05, "normalized": [ 2.2240740776654793, 1.123463076357621, 0.8892756502185444, 0.8391508499567661, 1.020497607750464, 0.9770305811455078, 0.8809641928112601, 0.815618216100643, 1.047627354359627, 1.038392580788716, 0.9776631042995007, 0.8263946439643135, 1.1834865254731135, 1.1158848074227978, 0.9831938132537951, 0.9253651743484881, 1.2047165912881532, 1.0585100087162846, 1.0183273795435872, 0.8939052877072882, 1.565309272455062, 1.3951478097472831, 1.2557330331978191, 1.0806924055185287, 0.9345587629892653, 0.8411301445051553, 0.7796712675898905, 0.7463030115246023, 0.6524170468838495, 0.5916843062269816, 0.5583702111657051, 0.5554412050239066 ], "peak_layer": 1, "peak_normalized": 2.2240740776654793, "population_cv": 0.31201770965801834, "rest_mean": 1.0769836350206148e-05, "spike_contrast": 1.3058465280826606, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4063753405935131e-05, "top_five_layers": [ 1, 21, 22, 23, 17 ], "top_five_spike_overlap": 3 }, "values": [ 2.5097588149947114e-05, 1.2677731319854502e-05, 1.0035040759248659e-05, 9.469406904827338e-06, 1.1515816368046217e-05, 1.102531223295955e-05, 9.941250027623028e-06, 9.203852641803678e-06, 1.1821962289104704e-05, 1.1717752386175562e-05, 1.1032449947379064e-05, 9.325459359388333e-06, 1.3355066585063469e-05, 1.2592214261530899e-05, 1.1094861292804126e-05, 1.0442293387313839e-05, 1.3594637493952177e-05, 1.1944767720706295e-05, 1.1491326404211577e-05, 1.00872839539079e-05, 1.7663749531493522e-05, 1.5743560652481392e-05, 1.4170333088259213e-05, 1.2195085218991153e-05, 1.0546038538450375e-05, 9.491742275713477e-06, 8.798208909865934e-06, 8.421664460911416e-06, 7.362207270489307e-06, 6.6768679971573874e-06, 6.30093472864246e-06, 6.267882326937979e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.889938600172172e-05, "first_to_last_ratio": 1.5413227470241377, "imbalance_abs_log_ratio": 0.43264097439819243, "last_quartile_mean": 3.172559809172526e-05, "mean": 4.484832186335552e-05, "normalized": [ 2.1812015108286227, 1.1117106638221061, 0.8712690674441437, 0.8155092170381977, 1.0253782370170077, 0.9956135898204023, 0.8876686512602217, 0.8342739405890812, 1.055082370640812, 1.0480930529822718, 0.9841224192619671, 0.8329693296087033, 1.1966925672041921, 1.1146712863155361, 0.9871271696154558, 0.9208744603675149, 1.199191469514801, 1.0575028324354707, 1.0341642237425372, 0.898469999535679, 1.5531345456552004, 1.3827320560469147, 1.2641281302304002, 1.089237902148047, 0.9325460360008964, 0.8434023238170155, 0.7808660077834011, 0.7497730633285921, 0.6533877584038497, 0.584941351024475, 0.5602379828066166, 0.5540267837098669 ], "peak_layer": 1, "peak_normalized": 2.1812015108286227, "population_cv": 0.30617930626208134, "rest_mean": 4.2818887676629754e-05, "spike_contrast": 1.303332933193729, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5807266471674664e-05, "top_five_layers": [ 1, 21, 22, 23, 17 ], "top_five_spike_overlap": 3 }, "values": [ 9.782322740647942e-05, 4.985835767001845e-05, 3.907495556632057e-05, 3.6574219848262146e-05, 4.598649320541881e-05, 4.465159872779623e-05, 3.981044937972911e-05, 3.741578620974906e-05, 4.73186737508513e-05, 4.700521458289586e-05, 4.413623901200481e-05, 3.73572765965946e-05, 5.366965342545882e-05, 4.999113662051968e-05, 4.42709970229771e-05, 4.1299674194306135e-05, 5.3781725000590086e-05, 4.742722740047611e-05, 4.638052996597253e-05, 4.029487172374502e-05, 6.965547800064087e-05, 6.201321230037138e-05, 5.6694025261094794e-05, 4.885049202130176e-05, 4.1823124774964526e-05, 3.782517887884751e-05, 3.502053004922345e-05, 3.362606366863474e-05, 2.9303344490472227e-05, 2.623363798193168e-05, 2.512573337298818e-05, 2.4847171516739763e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04336446523666382, "first_to_last_ratio": 1.541322758743648, "imbalance_abs_log_ratio": 0.4326409820017331, "last_quartile_mean": 0.02813457790762186, "mean": 0.03977194067556411, "normalized": [ 2.1812015617016365, 1.1117106345862457, 0.8712690698977081, 0.8155092439280437, 1.025378223837616, 0.9956136045129053, 0.8876686330275743, 0.834273974654176, 1.0550823347362368, 1.0480930495251253, 0.9841223425346584, 0.8329693905077741, 1.1966926583705826, 1.1146713324813935, 0.9871271572544008, 0.9208743646706732, 1.1991914877687524, 1.0575028591124085, 1.034164215905152, 0.8984699490823292, 1.553134589516788, 1.3827319963584066, 1.2641280815855902, 1.089237936269389, 0.9325460279316578, 0.8434023170272189, 0.7808660625001403, 0.7497730677902867, 0.653387781511157, 0.5849413243159465, 0.5602379632013756, 0.5540267638966512 ], "peak_layer": 1, "peak_normalized": 2.1812015617016365, "population_cv": 0.30617931423170164, "rest_mean": 0.03797221818455943, "spike_contrast": 1.3033329232031423, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.049490442126989366, "top_five_layers": [ 1, 21, 22, 23, 17 ], "top_five_spike_overlap": 3 }, "values": [ 0.08675061911344528, 0.0442148894071579, 0.03465206176042557, 0.032434385269880295, 0.040781281888484955, 0.03959748521447182, 0.03530430421233177, 0.03318069502711296, 0.041962672024965286, 0.04168469458818436, 0.039140455424785614, 0.03312880918383598, 0.047594789415597916, 0.04433264210820198, 0.03925996273756027, 0.0366249606013298, 0.04769417271018028, 0.04205894097685814, 0.04113071784377098, 0.035733893513679504, 0.061771176755428314, 0.05499393492937088, 0.050276827067136765, 0.04332110658288002, 0.037089165300130844, 0.033543746918439865, 0.031056558713316917, 0.029819929972290993, 0.025986500084400177, 0.02326425164937973, 0.022281751036643982, 0.022034719586372375 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.2504997307114536e-06, "first_to_last_ratio": 1.534048564025481, "imbalance_abs_log_ratio": 0.42791036086994516, "last_quartile_mean": 2.1189027563650598e-06, "mean": 2.8327239434133844e-06, "normalized": [ 2.434101831970979, 1.2717248047474439, 1.00501292981449, 0.978793620303735, 0.982603804852325, 0.9590124302490006, 0.7831434651115216, 0.7654630979136043, 0.9086838647658697, 0.9473256649897754, 0.8950270312608294, 0.7521042950228368, 1.0341349274760525, 1.0071135921377148, 0.8968409003212485, 0.9087337907131031, 1.1168468910837492, 0.9975784586172981, 0.9974908072758529, 0.913717074488086, 1.5359343623090123, 1.463151481657458, 1.3013629981753376, 1.160026574630551, 0.9743714014341565, 0.8914931250247432, 0.814663192509027, 0.7848098840673302, 0.7014159737591589, 0.636528977982725, 0.5932070215396675, 0.5875817237953173 ], "peak_layer": 1, "peak_normalized": 2.434101831970979, "population_cv": 0.3367092305215161, "rest_mean": 2.6821859827813804e-06, "spike_contrast": 1.3592006498541729, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6456289308262057e-06, "top_five_layers": [ 1, 21, 22, 23, 2 ], "top_five_spike_overlap": 3 }, "values": [ 6.895138540130574e-06, 3.6024453038407955e-06, 2.8469241897255415e-06, 2.772652123894659e-06, 2.7834453248942737e-06, 2.7166174731974024e-06, 2.2184292447491316e-06, 2.168345645259251e-06, 2.574050540715689e-06, 2.6835120934265433e-06, 2.535364501454751e-06, 2.1305038444552338e-06, 2.9294187697814777e-06, 2.8528747861855663e-06, 2.540502691772417e-06, 2.5741919671418145e-06, 3.163718929499737e-06, 2.8258643851586385e-06, 2.8256160931050545e-06, 2.588308234408032e-06, 4.350878043624107e-06, 4.144704234931851e-06, 3.686402124003507e-06, 3.2860350529517746e-06, 2.7601251986197894e-06, 2.525353920646012e-06, 2.307715931237908e-06, 2.2231497496250086e-06, 1.9869178231601836e-06, 1.803110876608116e-06, 1.6803917333163554e-06, 1.664456817707105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.123088797314267e-05, "first_to_last_ratio": 1.5481267051258016, "imbalance_abs_log_ratio": 0.4370456226743978, "last_quartile_mean": 2.663276063685771e-05, "mean": 3.721600086237231e-05, "normalized": [ 2.2576717646411373, 1.1550152331301582, 0.8940234958179116, 0.8176409477374584, 1.026845887840395, 0.9928503960506618, 0.8891512866739572, 0.8298463487568712, 1.0382073376210066, 1.010525228672297, 0.9673644078560969, 0.8327734670628495, 1.1597014181578864, 1.092703899157089, 0.9631469488595324, 0.9043154291215759, 1.184106843777168, 1.0436930451114852, 1.0042890707665433, 0.902293602113743, 1.5231038792179041, 1.386951430682882, 1.2813911026130895, 1.1173746266426017, 0.9589627259110588, 0.8812873444364125, 0.807412211801708, 0.7778540745678935, 0.6383882364209196, 0.5722421467697438, 0.546122523093635, 0.5427436389163276 ], "peak_layer": 1, "peak_normalized": 2.2576717646411373, "population_cv": 0.31425595490925345, "rest_mean": 3.5468525244073976e-05, "spike_contrast": 1.3153174224230797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.665236920118332e-05, "top_five_layers": [ 1, 21, 22, 23, 17 ], "top_five_spike_overlap": 3 }, "values": [ 8.402151433983818e-05, 4.298504791222513e-05, 3.3271979191340506e-05, 3.0429326216108166e-05, 3.82150974473916e-05, 3.694992119562812e-05, 3.309065505163744e-05, 3.0883562430972233e-05, 3.863792517222464e-05, 3.760770778171718e-05, 3.6001434637000784e-05, 3.099249806837179e-05, 4.31594489782583e-05, 4.0666069253347814e-05, 3.584447767934762e-05, 3.365500379004516e-05, 4.406772131915204e-05, 3.8842081266921014e-05, 3.7375622923718765e-05, 3.357975947437808e-05, 5.668383528245613e-05, 5.161678564036265e-05, 4.768825237988494e-05, 4.1584215068724006e-05, 3.5688757634488866e-05, 3.2797990570543334e-05, 3.00486535707023e-05, 2.894861790991854e-05, 2.3758257157169282e-05, 2.129656422766857e-05, 2.0324496290413663e-05, 2.0198747733957134e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.4926876878671465e-05, "first_to_last_ratio": 1.476240806784429, "imbalance_abs_log_ratio": 0.3894988610989889, "last_quartile_mean": 2.3659335738557274e-05, "mean": 3.1687308080563525e-05, "normalized": [ 2.2762986475001763, 1.1662144265070453, 0.8845459641933642, 0.8067880948230711, 1.022807332270146, 0.9684203821336351, 0.8774829326589954, 0.815326475621621, 1.0196651328217694, 0.9647280193706825, 0.9452574962733946, 0.828508179687011, 1.129511920443437, 1.0667478383346118, 0.9293458708194444, 0.8844500989246517, 1.1690667339751835, 1.0196011843730592, 0.9801960813926914, 0.9044117736621221, 1.5006890378974727, 1.3936090252586848, 1.313619848705842, 1.1595056946364453, 1.0169445109383177, 0.9550874175993482, 0.867249516139161, 0.8392884874733272, 0.6350726181832244, 0.5697204034365697, 0.5445274700537234, 0.5453113838917707 ], "peak_layer": 1, "peak_normalized": 2.2762986475001763, "population_cv": 0.3151213209355983, "rest_mean": 3.0062608116336548e-05, "spike_contrast": 1.3458808274656038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.046068788738921e-05, "top_five_layers": [ 1, 21, 22, 23, 17 ], "top_five_spike_overlap": 3 }, "values": [ 7.212977652670816e-05, 3.6954195820726454e-05, 2.8028880478814244e-05, 2.5564942916389555e-05, 3.2410011044703424e-05, 3.068663500016555e-05, 2.7805072022601962e-05, 2.583550121926237e-05, 3.231044320273213e-05, 3.0569633963750675e-05, 2.9952665499877185e-05, 2.62531939370092e-05, 3.579119220376015e-05, 3.380236739758402e-05, 2.9448468922055326e-05, 2.8025842766510323e-05, 3.704457776620984e-05, 3.230841684853658e-05, 3.1059775210451335e-05, 2.8658374503720552e-05, 4.755279587698169e-05, 4.415971852722578e-05, 4.162507684668526e-05, 3.674161416711286e-05, 3.222423401894048e-05, 3.0264149245340377e-05, 2.7480802600621246e-05, 2.6594792871037498e-05, 2.012374170590192e-05, 1.8052905943477526e-05, 1.7254609701922163e-05, 1.7279449821216986e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.852452538121725e-05, "first_to_last_ratio": 1.5689675763636528, "imbalance_abs_log_ratio": 0.4504178083771724, "last_quartile_mean": 5.6422150919388514e-05, "mean": 8.211212457354122e-05, "normalized": [ 2.1219723286224816, 1.09827042835026, 0.8643788556282443, 0.8279612862679215, 1.017320737753908, 0.9856182851753849, 0.8733076450463839, 0.8359162514841522, 1.0662813223328673, 1.0751002353169425, 1.0056898741580604, 0.8412786645166292, 1.2255342764451151, 1.1374637954311468, 1.0329797245885708, 0.9362220937013769, 1.2341718933129842, 1.085018444301826, 1.0615915480189244, 0.9105849975032602, 1.5381215664946966, 1.374354285922899, 1.2590874380574026, 1.0946904946303944, 0.8961975645628097, 0.7984172853485704, 0.7473454782550812, 0.7156455510623411, 0.6436881832738256, 0.5784219801847321, 0.5573798012661504, 0.5599876829846564 ], "peak_layer": 1, "peak_normalized": 2.1219723286224816, "population_cv": 0.303114029752154, "rest_mean": 7.85768893870732e-05, "spike_contrast": 1.2879409629202945, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010120239458046853, "top_five_layers": [ 1, 21, 22, 23, 17 ], "top_five_spike_overlap": 3 }, "values": [ 0.00017423965618945658, 9.018131822813302e-05, 7.09759842720814e-05, 6.7985660280101e-05, 8.353436714969575e-05, 8.093121141428128e-05, 7.17091461410746e-05, 6.863885937491432e-05, 8.755462476983666e-05, 8.827876445138827e-05, 8.257933222921565e-05, 6.907917850185186e-05, 0.000100631223176606, 9.339956886833534e-05, 8.482015982735902e-05, 7.687518518650904e-05, 0.00010134047624887899, 8.909316966310143e-05, 8.716953743714839e-05, 7.477006874978542e-05, 0.0001262984296772629, 0.00011285115033388138, 0.00010338634456275031, 8.98873622645624e-05, 7.358868606388569e-05, 6.555973959621042e-05, 6.136612500995398e-05, 5.8763176639331505e-05, 5.28546042914968e-05, 4.749545769300312e-05, 4.576763967634179e-05, 4.598177838488482e-05 ] } } } } } }, "step": 500 }, { "modes": { "learned": { "bits_per_byte": 2.064953725142933, "forward": { "activation_sha256": { "attention_branch_output": "3bc7ac091ccda73a60054802f3ec6d7bc68d1af6b8f7940e83f1843e7ab472be", "mlp_branch_output": "b5dfb9be1f653e88219f0cf5f3837c21e4150bbb11f0520e3acfd9e5ac781830", "post_attention_state": "e3e18e8967d646eb5fe3ca615db63633333413502a9ff39741999ecb78762644", "post_mlp_state": "1266324f6216c67f6f92c122a2698de3ea217c2cc1431a3715e69a04aa62551d", "pre_attention_input": "9ec6f7109f4e916c83054b81e32dd86c0125957a9aa2bade5062f6ae3f8e89ac", "pre_mlp_input": "f4a14b64dd8b186b4b509748e2f608d960603b5d908ec49d21001a2020554f02" }, "logits_sha256": "e65e488a65420b358dedf0b8a5957c28a5e46147110f919cae1df8502eac1276", "loss_nats": 1.43131685256958, "mixer_summary_sha256": "34a25dd049f4b5bf7ff812bb03925bb3c0f35a3d7d3518411476b9b87af79ff5" }, "loss_nats": 1.43131685256958, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.4580664336681366, "group": 1, "latest_source_mass": 0.18571940064430237, "layer": 1, "max_source_mass": 0.81428062915802, "mixer_index": 1, "normalized_entropy": 0.660850172250714, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.81428062915802, "median": 0.8348954319953918, "p05": 0.6508002281188965, "p95": 0.9081302285194397 }, { "label": "current_group_partial", "mean": 0.18571940064430237, "median": 0.16510456800460815, "p05": 0.0918697863817215, "p95": 0.3491998016834259 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6350213885307312, "group": 1, "latest_source_mass": 0.48109376430511475, "layer": 2, "max_source_mass": 0.5189062356948853, "mixer_index": 2, "normalized_entropy": 0.9161422080917097, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5189062356948853, "median": 0.49609383940696716, "p05": 0.2741084098815918, "p95": 0.8060141801834106 }, { "label": "current_group_partial", "mean": 0.48109376430511475, "median": 0.5039061903953552, "p05": 0.19398583471775055, "p95": 0.7258915901184082 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4277062714099884, "group": 1, "latest_source_mass": 0.15978552401065826, "layer": 2, "max_source_mass": 0.8402144908905029, "mixer_index": 3, "normalized_entropy": 0.6170497167202993, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8402144908905029, "median": 0.8553081154823303, "p05": 0.7220970392227173, "p95": 0.9125142097473145 }, { "label": "current_group_partial", "mean": 0.15978552401065826, "median": 0.14469186961650848, "p05": 0.08748584985733032, "p95": 0.2779030203819275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6868079900741577, "group": 1, "latest_source_mass": 0.5064847469329834, "layer": 3, "max_source_mass": 0.5064847469329834, "mixer_index": 4, "normalized_entropy": 0.9908544813229038, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4935153126716614, "median": 0.488329142332077, "p05": 0.41239702701568604, "p95": 0.5894505381584167 }, { "label": "current_group_partial", "mean": 0.5064847469329834, "median": 0.5116708278656006, "p05": 0.41054946184158325, "p95": 0.587602972984314 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.38686758279800415, "group": 1, "latest_source_mass": 0.13613657653331757, "layer": 3, "max_source_mass": 0.863863468170166, "mixer_index": 5, "normalized_entropy": 0.5581319431833811, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.863863468170166, "median": 0.8749346137046814, "p05": 0.7766229510307312, "p95": 0.9342007040977478 }, { "label": "current_group_partial", "mean": 0.13613657653331757, "median": 0.1250653862953186, "p05": 0.0657993033528328, "p95": 0.22337706387043 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6772255897521973, "group": 1, "latest_source_mass": 0.43206995725631714, "layer": 4, "max_source_mass": 0.5679300427436829, "mixer_index": 6, "normalized_entropy": 0.9770299998985986, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5679300427436829, "median": 0.5676445364952087, "p05": 0.476777583360672, "p95": 0.6623597741127014 }, { "label": "current_group_partial", "mean": 0.43206995725631714, "median": 0.43235546350479126, "p05": 0.3376401960849762, "p95": 0.5232223868370056 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5690979361534119, "group": 1, "latest_source_mass": 0.28186240792274475, "layer": 4, "max_source_mass": 0.7181376218795776, "mixer_index": 7, "normalized_entropy": 0.8210347702686712, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7181376218795776, "median": 0.7203682661056519, "p05": 0.5602526664733887, "p95": 0.9003764390945435 }, { "label": "current_group_partial", "mean": 0.28186240792274475, "median": 0.2796317934989929, "p05": 0.09962356090545654, "p95": 0.43974730372428894 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6592018008232117, "group": 2, "latest_source_mass": 0.4827212691307068, "layer": 5, "max_source_mass": 0.517278790473938, "mixer_index": 8, "normalized_entropy": 0.9510271689927217, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.517278790473938, "median": 0.5070376396179199, "p05": 0.33305567502975464, "p95": 0.7351903915405273 }, { "label": "completed_group_1", "mean": 0.4827212691307068, "median": 0.4929623603820801, "p05": 0.26480966806411743, "p95": 0.6669443845748901 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.038456916809082, "group": 2, "latest_source_mass": 0.3147233724594116, "layer": 5, "max_source_mass": 0.4540913999080658, "mixer_index": 9, "normalized_entropy": 0.9452442208415888, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4540913999080658, "median": 0.4537813663482666, "p05": 0.2934587895870209, "p95": 0.613494873046875 }, { "label": "completed_group_1", "mean": 0.23118522763252258, "median": 0.23724967241287231, "p05": 0.17357462644577026, "p95": 0.2779788076877594 }, { "label": "current_group_partial", "mean": 0.3147233724594116, "median": 0.31569617986679077, "p05": 0.190863698720932, "p95": 0.4443512260913849 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0903832912445068, "group": 2, "latest_source_mass": 0.38192373514175415, "layer": 6, "max_source_mass": 0.38192373514175415, "mixer_index": 10, "normalized_entropy": 0.9925096437492255, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3152569830417633, "median": 0.3122430443763733, "p05": 0.2715557813644409, "p95": 0.3724803924560547 }, { "label": "completed_group_1", "mean": 0.30281925201416016, "median": 0.303215354681015, "p05": 0.2769315242767334, "p95": 0.3273058235645294 }, { "label": "current_group_partial", "mean": 0.38192373514175415, "median": 0.3863401412963867, "p05": 0.32811740040779114, "p95": 0.4194748103618622 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0411981344223022, "group": 2, "latest_source_mass": 0.31767070293426514, "layer": 6, "max_source_mass": 0.4020236134529114, "mixer_index": 11, "normalized_entropy": 0.9477393846418622, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4020236134529114, "median": 0.386812299489975, "p05": 0.2018219530582428, "p95": 0.707051694393158 }, { "label": "completed_group_1", "mean": 0.2803056538105011, "median": 0.29224973917007446, "p05": 0.12760642170906067, "p95": 0.363227903842926 }, { "label": "current_group_partial", "mean": 0.31767070293426514, "median": 0.32260167598724365, "p05": 0.1550494283437729, "p95": 0.4480850100517273 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0905810594558716, "group": 2, "latest_source_mass": 0.384937047958374, "layer": 7, "max_source_mass": 0.384937047958374, "mixer_index": 12, "normalized_entropy": 0.9926896601329894, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3278552293777466, "median": 0.32747673988342285, "p05": 0.305451899766922, "p95": 0.3515099585056305 }, { "label": "completed_group_1", "mean": 0.2872077226638794, "median": 0.28588157892227173, "p05": 0.26596081256866455, "p95": 0.30896520614624023 }, { "label": "current_group_partial", "mean": 0.384937047958374, "median": 0.38384974002838135, "p05": 0.3644270896911621, "p95": 0.4090334475040436 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0150185823440552, "group": 2, "latest_source_mass": 0.29576438665390015, "layer": 7, "max_source_mass": 0.48287349939346313, "mixer_index": 13, "normalized_entropy": 0.9239097294047216, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48287349939346313, "median": 0.4650489091873169, "p05": 0.3346910774707794, "p95": 0.7498793601989746 }, { "label": "completed_group_1", "mean": 0.2213621288537979, "median": 0.22629523277282715, "p05": 0.10256442427635193, "p95": 0.29524046182632446 }, { "label": "current_group_partial", "mean": 0.29576438665390015, "median": 0.3073716163635254, "p05": 0.14070476591587067, "p95": 0.3917995095252991 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0816482305526733, "group": 2, "latest_source_mass": 0.40547415614128113, "layer": 8, "max_source_mass": 0.40547415614128113, "mixer_index": 14, "normalized_entropy": 0.9845586488605524, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3306047320365906, "median": 0.33274394273757935, "p05": 0.2933180034160614, "p95": 0.3644672632217407 }, { "label": "completed_group_1", "mean": 0.2639211118221283, "median": 0.2629454731941223, "p05": 0.23683591187000275, "p95": 0.2942429184913635 }, { "label": "current_group_partial", "mean": 0.40547415614128113, "median": 0.40589621663093567, "p05": 0.37255728244781494, "p95": 0.43844079971313477 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0114538669586182, "group": 2, "latest_source_mass": 0.3466050922870636, "layer": 8, "max_source_mass": 0.4296638071537018, "mixer_index": 15, "normalized_entropy": 0.9206649856291366, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4296638071537018, "median": 0.39453762769699097, "p05": 0.24606716632843018, "p95": 0.7861940860748291 }, { "label": "completed_group_1", "mean": 0.22373110055923462, "median": 0.22628825902938843, "p05": 0.07940524816513062, "p95": 0.3281305432319641 }, { "label": "current_group_partial", "mean": 0.3466050922870636, "median": 0.36169061064720154, "p05": 0.12893560528755188, "p95": 0.47660863399505615 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0918333530426025, "group": 3, "latest_source_mass": 0.32777342200279236, "layer": 9, "max_source_mass": 0.36806726455688477, "mixer_index": 16, "normalized_entropy": 0.9938295468788851, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.36806726455688477, "median": 0.3683367669582367, "p05": 0.31737297773361206, "p95": 0.4160827398300171 }, { "label": "completed_group_1", "mean": 0.3041592836380005, "median": 0.3026740550994873, "p05": 0.27596747875213623, "p95": 0.3395172357559204 }, { "label": "completed_group_2", "mean": 0.32777342200279236, "median": 0.3294421434402466, "p05": 0.2683165669441223, "p95": 0.3802773654460907 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3290088176727295, "group": 3, "latest_source_mass": 0.2603103518486023, "layer": 9, "max_source_mass": 0.35130923986434937, "mixer_index": 17, "normalized_entropy": 0.9586772152770757, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.35130923986434937, "median": 0.3600649833679199, "p05": 0.20772139728069305, "p95": 0.5109238028526306 }, { "label": "completed_group_1", "mean": 0.17179939150810242, "median": 0.16660934686660767, "p05": 0.11614341288805008, "p95": 0.23857468366622925 }, { "label": "completed_group_2", "mean": 0.21658103168010712, "median": 0.21380692720413208, "p05": 0.1328548789024353, "p95": 0.30206120014190674 }, { "label": "current_group_partial", "mean": 0.2603103518486023, "median": 0.25967299938201904, "p05": 0.21334272623062134, "p95": 0.3113977313041687 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.379033088684082, "group": 3, "latest_source_mass": 0.24449527263641357, "layer": 10, "max_source_mass": 0.29040855169296265, "mixer_index": 18, "normalized_entropy": 0.9947620991331576, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.29040855169296265, "median": 0.29490548372268677, "p05": 0.2500562369823456, "p95": 0.32532158493995667 }, { "label": "completed_group_1", "mean": 0.23445811867713928, "median": 0.2324211150407791, "p05": 0.21483668684959412, "p95": 0.26111143827438354 }, { "label": "completed_group_2", "mean": 0.2306380271911621, "median": 0.23122423887252808, "p05": 0.19837205111980438, "p95": 0.26181501150131226 }, { "label": "current_group_partial", "mean": 0.24449527263641357, "median": 0.2435547560453415, "p05": 0.21777315437793732, "p95": 0.2725067138671875 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3394957780838013, "group": 3, "latest_source_mass": 0.26184359192848206, "layer": 10, "max_source_mass": 0.3262045383453369, "mixer_index": 19, "normalized_entropy": 0.9662419581666017, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3262045383453369, "median": 0.33913201093673706, "p05": 0.15982496738433838, "p95": 0.49248045682907104 }, { "label": "completed_group_1", "mean": 0.18669261038303375, "median": 0.1907009780406952, "p05": 0.1366541087627411, "p95": 0.22178497910499573 }, { "label": "completed_group_2", "mean": 0.22525924444198608, "median": 0.22064685821533203, "p05": 0.14132791757583618, "p95": 0.31671440601348877 }, { "label": "current_group_partial", "mean": 0.26184359192848206, "median": 0.2529178857803345, "p05": 0.2070307433605194, "p95": 0.33866065740585327 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3677098751068115, "group": 3, "latest_source_mass": 0.2845887541770935, "layer": 11, "max_source_mass": 0.29889947175979614, "mixer_index": 20, "normalized_entropy": 0.9865941270957302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29889947175979614, "median": 0.2991558313369751, "p05": 0.25637567043304443, "p95": 0.345245897769928 }, { "label": "completed_group_1", "mean": 0.22409284114837646, "median": 0.2220568209886551, "p05": 0.19557906687259674, "p95": 0.2607859671115875 }, { "label": "completed_group_2", "mean": 0.1924189329147339, "median": 0.1933085322380066, "p05": 0.16921265423297882, "p95": 0.21061491966247559 }, { "label": "current_group_partial", "mean": 0.2845887541770935, "median": 0.286131888628006, "p05": 0.25166112184524536, "p95": 0.31440335512161255 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2910308837890625, "group": 3, "latest_source_mass": 0.2587660849094391, "layer": 11, "max_source_mass": 0.3990757167339325, "mixer_index": 21, "normalized_entropy": 0.9312819268384881, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3990757167339325, "median": 0.4163215160369873, "p05": 0.22181591391563416, "p95": 0.5799376368522644 }, { "label": "completed_group_1", "mean": 0.18164023756980896, "median": 0.17469573020935059, "p05": 0.11628367006778717, "p95": 0.24950891733169556 }, { "label": "completed_group_2", "mean": 0.16051796078681946, "median": 0.15820392966270447, "p05": 0.09586596488952637, "p95": 0.21985307335853577 }, { "label": "current_group_partial", "mean": 0.2587660849094391, "median": 0.2571001946926117, "p05": 0.18264421820640564, "p95": 0.3371993899345398 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3623619079589844, "group": 3, "latest_source_mass": 0.29877203702926636, "layer": 12, "max_source_mass": 0.3024129569530487, "mixer_index": 22, "normalized_entropy": 0.9827363842542266, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3024129569530487, "median": 0.3032229542732239, "p05": 0.26408907771110535, "p95": 0.3383038640022278 }, { "label": "completed_group_1", "mean": 0.21113446354866028, "median": 0.20936083793640137, "p05": 0.19063444435596466, "p95": 0.236852765083313 }, { "label": "completed_group_2", "mean": 0.18768057227134705, "median": 0.1870611608028412, "p05": 0.16947540640830994, "p95": 0.20746007561683655 }, { "label": "current_group_partial", "mean": 0.29877203702926636, "median": 0.30005669593811035, "p05": 0.26102375984191895, "p95": 0.332477867603302 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2281851768493652, "group": 3, "latest_source_mass": 0.23528897762298584, "layer": 12, "max_source_mass": 0.46696996688842773, "mixer_index": 23, "normalized_entropy": 0.8859483319669569, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.46696996688842773, "median": 0.4326247572898865, "p05": 0.3197334408760071, "p95": 0.6868066787719727 }, { "label": "completed_group_1", "mean": 0.16045212745666504, "median": 0.16051796078681946, "p05": 0.08773931860923767, "p95": 0.22867971658706665 }, { "label": "completed_group_2", "mean": 0.13728895783424377, "median": 0.14161056280136108, "p05": 0.08086587488651276, "p95": 0.1866428703069687 }, { "label": "current_group_partial", "mean": 0.23528897762298584, "median": 0.24016538262367249, "p05": 0.13368576765060425, "p95": 0.32655879855155945 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3772211074829102, "group": 4, "latest_source_mass": 0.24272704124450684, "layer": 13, "max_source_mass": 0.28748467564582825, "mixer_index": 24, "normalized_entropy": 0.9934550309866003, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.28748467564582825, "median": 0.293403685092926, "p05": 0.22645097970962524, "p95": 0.3287804424762726 }, { "label": "completed_group_1", "mean": 0.24638661742210388, "median": 0.247311532497406, "p05": 0.21059969067573547, "p95": 0.27817288041114807 }, { "label": "completed_group_2", "mean": 0.22340166568756104, "median": 0.22116877138614655, "p05": 0.19812123477458954, "p95": 0.2556689977645874 }, { "label": "completed_group_3", "mean": 0.24272704124450684, "median": 0.24190634489059448, "p05": 0.20034869015216827, "p95": 0.2857007384300232 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5519046783447266, "group": 4, "latest_source_mass": 0.23496176302433014, "layer": 13, "max_source_mass": 0.30400484800338745, "mixer_index": 25, "normalized_entropy": 0.9642525917620761, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.30400484800338745, "median": 0.3028053641319275, "p05": 0.22878815233707428, "p95": 0.400842547416687 }, { "label": "completed_group_1", "mean": 0.1470816731452942, "median": 0.1489969789981842, "p05": 0.11740177869796753, "p95": 0.17312948405742645 }, { "label": "completed_group_2", "mean": 0.13316048681735992, "median": 0.13380160927772522, "p05": 0.10824885964393616, "p95": 0.15745775401592255 }, { "label": "completed_group_3", "mean": 0.1807912290096283, "median": 0.17882078886032104, "p05": 0.14052768051624298, "p95": 0.22473205626010895 }, { "label": "current_group_partial", "mean": 0.23496176302433014, "median": 0.23252201080322266, "p05": 0.20341888070106506, "p95": 0.27739572525024414 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.572178602218628, "group": 4, "latest_source_mass": 0.28295373916625977, "layer": 14, "max_source_mass": 0.28295373916625977, "mixer_index": 26, "normalized_entropy": 0.9768494889255332, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21885336935520172, "median": 0.2175014615058899, "p05": 0.1686568409204483, "p95": 0.2588198482990265 }, { "label": "completed_group_1", "mean": 0.2029750645160675, "median": 0.20364594459533691, "p05": 0.16090315580368042, "p95": 0.2408662736415863 }, { "label": "completed_group_2", "mean": 0.1456144154071808, "median": 0.1434675008058548, "p05": 0.12856581807136536, "p95": 0.16848507523536682 }, { "label": "completed_group_3", "mean": 0.14960341155529022, "median": 0.14882786571979523, "p05": 0.11882077157497406, "p95": 0.17920038104057312 }, { "label": "current_group_partial", "mean": 0.28295373916625977, "median": 0.28187495470046997, "p05": 0.24177348613739014, "p95": 0.3278854489326477 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5391805171966553, "group": 4, "latest_source_mass": 0.2634100914001465, "layer": 14, "max_source_mass": 0.2898171544075012, "mixer_index": 27, "normalized_entropy": 0.9563466259278133, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2898171544075012, "median": 0.2940465807914734, "p05": 0.18881390988826752, "p95": 0.4135587215423584 }, { "label": "completed_group_1", "mean": 0.15984712541103363, "median": 0.15493696928024292, "p05": 0.1193767637014389, "p95": 0.2155456840991974 }, { "label": "completed_group_2", "mean": 0.1228363886475563, "median": 0.12368190288543701, "p05": 0.08640466630458832, "p95": 0.15742656588554382 }, { "label": "completed_group_3", "mean": 0.16408920288085938, "median": 0.16305197775363922, "p05": 0.12961797416210175, "p95": 0.20428991317749023 }, { "label": "current_group_partial", "mean": 0.2634100914001465, "median": 0.26288318634033203, "p05": 0.2051006406545639, "p95": 0.3223395347595215 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5521924495697021, "group": 4, "latest_source_mass": 0.32056212425231934, "layer": 15, "max_source_mass": 0.32056212425231934, "mixer_index": 28, "normalized_entropy": 0.9644313940773145, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.210292249917984, "median": 0.20937448740005493, "p05": 0.16961288452148438, "p95": 0.2644428610801697 }, { "label": "completed_group_1", "mean": 0.185291588306427, "median": 0.1801799237728119, "p05": 0.1599384844303131, "p95": 0.23004677891731262 }, { "label": "completed_group_2", "mean": 0.14173434674739838, "median": 0.13837604224681854, "p05": 0.12188456207513809, "p95": 0.17588482797145844 }, { "label": "completed_group_3", "mean": 0.14211969077587128, "median": 0.14116454124450684, "p05": 0.1147841066122055, "p95": 0.17095403373241425 }, { "label": "current_group_partial", "mean": 0.32056212425231934, "median": 0.3242885172367096, "p05": 0.25657448172569275, "p95": 0.3705737292766571 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5468778610229492, "group": 4, "latest_source_mass": 0.26227355003356934, "layer": 15, "max_source_mass": 0.2756267488002777, "mixer_index": 29, "normalized_entropy": 0.9611292545504065, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2756267488002777, "median": 0.26299941539764404, "p05": 0.16425450146198273, "p95": 0.411077082157135 }, { "label": "completed_group_1", "mean": 0.16966477036476135, "median": 0.16762341558933258, "p05": 0.1361422836780548, "p95": 0.2147567719221115 }, { "label": "completed_group_2", "mean": 0.133374884724617, "median": 0.13428664207458496, "p05": 0.09884629398584366, "p95": 0.16465862095355988 }, { "label": "completed_group_3", "mean": 0.1590600460767746, "median": 0.1579585075378418, "p05": 0.12251652777194977, "p95": 0.20119814574718475 }, { "label": "current_group_partial", "mean": 0.26227355003356934, "median": 0.2576714754104614, "p05": 0.1876349151134491, "p95": 0.35171017050743103 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5309922695159912, "group": 4, "latest_source_mass": 0.2753770351409912, "layer": 16, "max_source_mass": 0.2753770351409912, "mixer_index": 30, "normalized_entropy": 0.95125898159099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21606208384037018, "median": 0.21523752808570862, "p05": 0.17893150448799133, "p95": 0.25598081946372986 }, { "label": "completed_group_1", "mean": 0.27386677265167236, "median": 0.2663733959197998, "p05": 0.20507416129112244, "p95": 0.368195503950119 }, { "label": "completed_group_2", "mean": 0.12764990329742432, "median": 0.12438307702541351, "p05": 0.10569629073143005, "p95": 0.15790577232837677 }, { "label": "completed_group_3", "mean": 0.10704416781663895, "median": 0.10739792883396149, "p05": 0.07143788039684296, "p95": 0.13995352387428284 }, { "label": "current_group_partial", "mean": 0.2753770351409912, "median": 0.278347373008728, "p05": 0.2043362706899643, "p95": 0.333025723695755 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5832164287567139, "group": 4, "latest_source_mass": 0.1685337871313095, "layer": 16, "max_source_mass": 0.24636118113994598, "mixer_index": 31, "normalized_entropy": 0.9837076761552552, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22763225436210632, "median": 0.2050790637731552, "p05": 0.15834280848503113, "p95": 0.3526703417301178 }, { "label": "completed_group_1", "mean": 0.24636118113994598, "median": 0.24476206302642822, "p05": 0.2120620459318161, "p95": 0.2850618064403534 }, { "label": "completed_group_2", "mean": 0.17577087879180908, "median": 0.17898494005203247, "p05": 0.13335087895393372, "p95": 0.20948299765586853 }, { "label": "completed_group_3", "mean": 0.1817018985748291, "median": 0.17961755394935608, "p05": 0.14637041091918945, "p95": 0.22162409126758575 }, { "label": "current_group_partial", "mean": 0.1685337871313095, "median": 0.1639421284198761, "p05": 0.12563997507095337, "p95": 0.22547563910484314 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5793015956878662, "group": 5, "latest_source_mass": 0.25108590722084045, "layer": 17, "max_source_mass": 0.25108590722084045, "mixer_index": 32, "normalized_entropy": 0.9812752536066109, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2158121019601822, "median": 0.21408912539482117, "p05": 0.1828576922416687, "p95": 0.25927743315696716 }, { "label": "completed_group_1", "mean": 0.21387425065040588, "median": 0.2010079026222229, "p05": 0.15640759468078613, "p95": 0.3137179911136627 }, { "label": "completed_group_2", "mean": 0.1464191973209381, "median": 0.14469388127326965, "p05": 0.1298638880252838, "p95": 0.16936294734477997 }, { "label": "completed_group_3", "mean": 0.17280855774879456, "median": 0.17517459392547607, "p05": 0.13696318864822388, "p95": 0.198440819978714 }, { "label": "completed_group_4", "mean": 0.25108590722084045, "median": 0.2582312226295471, "p05": 0.1583583801984787, "p95": 0.3231808543205261 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7719972133636475, "group": 5, "latest_source_mass": 0.1918708086013794, "layer": 17, "max_source_mass": 0.20328448712825775, "mixer_index": 33, "normalized_entropy": 0.9889704749974495, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16962553560733795, "median": 0.1690855771303177, "p05": 0.1343761533498764, "p95": 0.2083701342344284 }, { "label": "completed_group_1", "mean": 0.20328448712825775, "median": 0.20764055848121643, "p05": 0.16696703433990479, "p95": 0.2276618480682373 }, { "label": "completed_group_2", "mean": 0.1659279465675354, "median": 0.16634680330753326, "p05": 0.1471528857946396, "p95": 0.18308526277542114 }, { "label": "completed_group_3", "mean": 0.14008209109306335, "median": 0.1373065710067749, "p05": 0.10879024863243103, "p95": 0.18058663606643677 }, { "label": "completed_group_4", "mean": 0.12920911610126495, "median": 0.1269148886203766, "p05": 0.10906878858804703, "p95": 0.1541941612958908 }, { "label": "current_group_partial", "mean": 0.1918708086013794, "median": 0.19034206867218018, "p05": 0.14745062589645386, "p95": 0.23773568868637085 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.689546823501587, "group": 5, "latest_source_mass": 0.3272479772567749, "layer": 18, "max_source_mass": 0.3272479772567749, "mixer_index": 34, "normalized_entropy": 0.9429540362521402, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16601097583770752, "median": 0.16331082582473755, "p05": 0.13003051280975342, "p95": 0.20780879259109497 }, { "label": "completed_group_1", "mean": 0.1523149609565735, "median": 0.15154623985290527, "p05": 0.1387515515089035, "p95": 0.16826456785202026 }, { "label": "completed_group_2", "mean": 0.0875597670674324, "median": 0.08667673170566559, "p05": 0.0783165767788887, "p95": 0.09952021390199661 }, { "label": "completed_group_3", "mean": 0.10126751661300659, "median": 0.0997447669506073, "p05": 0.08770889788866043, "p95": 0.12069930881261826 }, { "label": "completed_group_4", "mean": 0.1655988097190857, "median": 0.16505587100982666, "p05": 0.15017777681350708, "p95": 0.18333116173744202 }, { "label": "current_group_partial", "mean": 0.3272479772567749, "median": 0.32916754484176636, "p05": 0.2944076955318451, "p95": 0.355704128742218 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5251271724700928, "group": 5, "latest_source_mass": 0.4654960334300995, "layer": 18, "max_source_mass": 0.4654960334300995, "mixer_index": 35, "normalized_entropy": 0.8511896817976157, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13745036721229553, "median": 0.12849631905555725, "p05": 0.08826753497123718, "p95": 0.21900935471057892 }, { "label": "completed_group_1", "mean": 0.11702707409858704, "median": 0.11415569484233856, "p05": 0.09654439240694046, "p95": 0.14336207509040833 }, { "label": "completed_group_2", "mean": 0.07592720538377762, "median": 0.07502187043428421, "p05": 0.061439283192157745, "p95": 0.09369536489248276 }, { "label": "completed_group_3", "mean": 0.08887732028961182, "median": 0.08338192105293274, "p05": 0.0636318027973175, "p95": 0.1313587874174118 }, { "label": "completed_group_4", "mean": 0.11522199958562851, "median": 0.11553025990724564, "p05": 0.09672605991363525, "p95": 0.13243958353996277 }, { "label": "current_group_partial", "mean": 0.4654960334300995, "median": 0.47634756565093994, "p05": 0.36450207233428955, "p95": 0.5321885943412781 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7189464569091797, "group": 5, "latest_source_mass": 0.29706352949142456, "layer": 19, "max_source_mass": 0.29706352949142456, "mixer_index": 36, "normalized_entropy": 0.9593622840736289, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1744518131017685, "median": 0.1729438602924347, "p05": 0.14070478081703186, "p95": 0.20897650718688965 }, { "label": "completed_group_1", "mean": 0.15982553362846375, "median": 0.15947404503822327, "p05": 0.14682501554489136, "p95": 0.1730521321296692 }, { "label": "completed_group_2", "mean": 0.09849169850349426, "median": 0.09798875451087952, "p05": 0.09000753611326218, "p95": 0.1088048666715622 }, { "label": "completed_group_3", "mean": 0.10780535638332367, "median": 0.10671155154705048, "p05": 0.09614568203687668, "p95": 0.12314765155315399 }, { "label": "completed_group_4", "mean": 0.16236206889152527, "median": 0.16202759742736816, "p05": 0.14780421555042267, "p95": 0.17789581418037415 }, { "label": "current_group_partial", "mean": 0.29706352949142456, "median": 0.297796368598938, "p05": 0.2712000906467438, "p95": 0.32343345880508423 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5877950191497803, "group": 5, "latest_source_mass": 0.4179171919822693, "layer": 19, "max_source_mass": 0.4179171919822693, "mixer_index": 37, "normalized_entropy": 0.8861652729726335, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14238224923610687, "median": 0.13146832585334778, "p05": 0.08928614854812622, "p95": 0.2557840645313263 }, { "label": "completed_group_1", "mean": 0.1503014862537384, "median": 0.14734625816345215, "p05": 0.12638163566589355, "p95": 0.1823616921901703 }, { "label": "completed_group_2", "mean": 0.10275131464004517, "median": 0.10360147058963776, "p05": 0.08169187605381012, "p95": 0.1201353371143341 }, { "label": "completed_group_3", "mean": 0.10159259289503098, "median": 0.09856455028057098, "p05": 0.08053024113178253, "p95": 0.1288338005542755 }, { "label": "completed_group_4", "mean": 0.08505518734455109, "median": 0.0846521332859993, "p05": 0.06617184728384018, "p95": 0.10538893193006516 }, { "label": "current_group_partial", "mean": 0.4179171919822693, "median": 0.42233192920684814, "p05": 0.3364676535129547, "p95": 0.48611512780189514 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7369492053985596, "group": 5, "latest_source_mass": 0.2724200189113617, "layer": 20, "max_source_mass": 0.2724200189113617, "mixer_index": 38, "normalized_entropy": 0.9694098093126812, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.17980656027793884, "median": 0.18241192400455475, "p05": 0.1317644566297531, "p95": 0.2213428020477295 }, { "label": "completed_group_1", "mean": 0.1706826537847519, "median": 0.17070558667182922, "p05": 0.15632681548595428, "p95": 0.1849895715713501 }, { "label": "completed_group_2", "mean": 0.10907261073589325, "median": 0.10894737392663956, "p05": 0.09957940876483917, "p95": 0.11852658540010452 }, { "label": "completed_group_3", "mean": 0.1106589287519455, "median": 0.10897988080978394, "p05": 0.097733274102211, "p95": 0.12891732156276703 }, { "label": "completed_group_4", "mean": 0.15735921263694763, "median": 0.15671271085739136, "p05": 0.14351971447467804, "p95": 0.17215019464492798 }, { "label": "current_group_partial", "mean": 0.2724200189113617, "median": 0.27173393964767456, "p05": 0.24179266393184662, "p95": 0.3049001395702362 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5987625122070312, "group": 5, "latest_source_mass": 0.39448028802871704, "layer": 20, "max_source_mass": 0.39448028802871704, "mixer_index": 39, "normalized_entropy": 0.8922863473945123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15493065118789673, "median": 0.1367141604423523, "p05": 0.08384928107261658, "p95": 0.33080407977104187 }, { "label": "completed_group_1", "mean": 0.15333226323127747, "median": 0.14968673884868622, "p05": 0.1258050799369812, "p95": 0.1917010396718979 }, { "label": "completed_group_2", "mean": 0.10736657679080963, "median": 0.10799242556095123, "p05": 0.07847796380519867, "p95": 0.1328052282333374 }, { "label": "completed_group_3", "mean": 0.1186048835515976, "median": 0.1159338653087616, "p05": 0.08940468728542328, "p95": 0.15236710011959076 }, { "label": "completed_group_4", "mean": 0.07128533720970154, "median": 0.06884299218654633, "p05": 0.04878491908311844, "p95": 0.10267870128154755 }, { "label": "current_group_partial", "mean": 0.39448028802871704, "median": 0.4018329381942749, "p05": 0.2799415588378906, "p95": 0.47351574897766113 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.75795578956604, "group": 6, "latest_source_mass": 0.2277819961309433, "layer": 21, "max_source_mass": 0.2277819961309433, "mixer_index": 40, "normalized_entropy": 0.9811338071640952, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1946164071559906, "median": 0.19585168361663818, "p05": 0.1496070772409439, "p95": 0.23593875765800476 }, { "label": "completed_group_1", "mean": 0.17554128170013428, "median": 0.17525508999824524, "p05": 0.16232231259346008, "p95": 0.1893068552017212 }, { "label": "completed_group_2", "mean": 0.11572675406932831, "median": 0.11579035222530365, "p05": 0.10456694662570953, "p95": 0.126620814204216 }, { "label": "completed_group_3", "mean": 0.1155121773481369, "median": 0.11377307772636414, "p05": 0.1026163250207901, "p95": 0.13427132368087769 }, { "label": "completed_group_4", "mean": 0.1708213835954666, "median": 0.17137710750102997, "p05": 0.15489280223846436, "p95": 0.18481197953224182 }, { "label": "completed_group_5", "mean": 0.2277819961309433, "median": 0.22828495502471924, "p05": 0.1998254656791687, "p95": 0.25523197650909424 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5344648361206055, "group": 6, "latest_source_mass": 0.5061697959899902, "layer": 21, "max_source_mass": 0.5061697959899902, "mixer_index": 41, "normalized_entropy": 0.7885589357070504, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0905577763915062, "median": 0.07774583250284195, "p05": 0.049066200852394104, "p95": 0.19859620928764343 }, { "label": "completed_group_1", "mean": 0.0786280632019043, "median": 0.07735998928546906, "p05": 0.06383179128170013, "p95": 0.09802225232124329 }, { "label": "completed_group_2", "mean": 0.06272733211517334, "median": 0.062279097735881805, "p05": 0.04816657304763794, "p95": 0.07942435145378113 }, { "label": "completed_group_3", "mean": 0.07359343767166138, "median": 0.07242873311042786, "p05": 0.056279584765434265, "p95": 0.09239646047353745 }, { "label": "completed_group_4", "mean": 0.050107162445783615, "median": 0.048957135528326035, "p05": 0.038674917072057724, "p95": 0.06534890085458755 }, { "label": "completed_group_5", "mean": 0.13821640610694885, "median": 0.13806892931461334, "p05": 0.10529474169015884, "p95": 0.17281517386436462 }, { "label": "current_group_partial", "mean": 0.5061697959899902, "median": 0.5132234692573547, "p05": 0.41420048475265503, "p95": 0.5726657509803772 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7138500213623047, "group": 6, "latest_source_mass": 0.4071047902107239, "layer": 22, "max_source_mass": 0.4071047902107239, "mixer_index": 42, "normalized_entropy": 0.8807446850484503, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13302071392536163, "median": 0.13715070486068726, "p05": 0.08387725800275803, "p95": 0.173191100358963 }, { "label": "completed_group_1", "mean": 0.10984362661838531, "median": 0.11014518141746521, "p05": 0.08778759092092514, "p95": 0.12984654307365417 }, { "label": "completed_group_2", "mean": 0.07243296504020691, "median": 0.07197447121143341, "p05": 0.06215493753552437, "p95": 0.08360114693641663 }, { "label": "completed_group_3", "mean": 0.07083482295274734, "median": 0.06908848881721497, "p05": 0.05891891568899155, "p95": 0.08817940950393677 }, { "label": "completed_group_4", "mean": 0.08885612338781357, "median": 0.08808258175849915, "p05": 0.07647298276424408, "p95": 0.10338804125785828 }, { "label": "completed_group_5", "mean": 0.11790697276592255, "median": 0.11741673201322556, "p05": 0.09878336638212204, "p95": 0.13930705189704895 }, { "label": "current_group_partial", "mean": 0.4071047902107239, "median": 0.4047229290008545, "p05": 0.34799832105636597, "p95": 0.4778216481208801 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4565383195877075, "group": 6, "latest_source_mass": 0.5493720769882202, "layer": 22, "max_source_mass": 0.5493720769882202, "mixer_index": 43, "normalized_entropy": 0.7485126280341451, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09418222308158875, "median": 0.08733601868152618, "p05": 0.04874149709939957, "p95": 0.1813039481639862 }, { "label": "completed_group_1", "mean": 0.07457256317138672, "median": 0.07398833334445953, "p05": 0.05570545420050621, "p95": 0.09413403272628784 }, { "label": "completed_group_2", "mean": 0.057202231138944626, "median": 0.05683571845293045, "p05": 0.04444708675146103, "p95": 0.07107516378164291 }, { "label": "completed_group_3", "mean": 0.06210128217935562, "median": 0.06029999628663063, "p05": 0.04672754555940628, "p95": 0.08132980763912201 }, { "label": "completed_group_4", "mean": 0.04410415142774582, "median": 0.043332912027835846, "p05": 0.03435574471950531, "p95": 0.05708703026175499 }, { "label": "completed_group_5", "mean": 0.11846547573804855, "median": 0.1162438839673996, "p05": 0.09147419780492783, "p95": 0.15436163544654846 }, { "label": "current_group_partial", "mean": 0.5493720769882202, "median": 0.5507825016975403, "p05": 0.4547572731971741, "p95": 0.6371357440948486 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7785513401031494, "group": 6, "latest_source_mass": 0.3567505478858948, "layer": 23, "max_source_mass": 0.3567505478858948, "mixer_index": 44, "normalized_entropy": 0.9139945854985072, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1471671462059021, "median": 0.15290190279483795, "p05": 0.09501753002405167, "p95": 0.18610240519046783 }, { "label": "completed_group_1", "mean": 0.12308044731616974, "median": 0.12249830365180969, "p05": 0.10776080191135406, "p95": 0.14060497283935547 }, { "label": "completed_group_2", "mean": 0.07605893164873123, "median": 0.07562456279993057, "p05": 0.06604670733213425, "p95": 0.08747582882642746 }, { "label": "completed_group_3", "mean": 0.07563325762748718, "median": 0.07386856526136398, "p05": 0.06278492510318756, "p95": 0.09318976104259491 }, { "label": "completed_group_4", "mean": 0.1090458557009697, "median": 0.10803408920764923, "p05": 0.094943106174469, "p95": 0.12811031937599182 }, { "label": "completed_group_5", "mean": 0.11226382851600647, "median": 0.1111060380935669, "p05": 0.09618639945983887, "p95": 0.13308405876159668 }, { "label": "current_group_partial", "mean": 0.3567505478858948, "median": 0.35415834188461304, "p05": 0.2961513102054596, "p95": 0.4284268319606781 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5135828256607056, "group": 6, "latest_source_mass": 0.5200446844100952, "layer": 23, "max_source_mass": 0.5200446844100952, "mixer_index": 45, "normalized_entropy": 0.77782770514636, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10121430456638336, "median": 0.08898357301950455, "p05": 0.06175067648291588, "p95": 0.20128758251667023 }, { "label": "completed_group_1", "mean": 0.07933001220226288, "median": 0.07804793119430542, "p05": 0.06303977221250534, "p95": 0.0987483337521553 }, { "label": "completed_group_2", "mean": 0.061728835105895996, "median": 0.06086552143096924, "p05": 0.04682660102844238, "p95": 0.0792360007762909 }, { "label": "completed_group_3", "mean": 0.06817515194416046, "median": 0.0653572827577591, "p05": 0.050377603620290756, "p95": 0.0938427746295929 }, { "label": "completed_group_4", "mean": 0.048567142337560654, "median": 0.04761301726102829, "p05": 0.03707665950059891, "p95": 0.06356129050254822 }, { "label": "completed_group_5", "mean": 0.12093988060951233, "median": 0.11864300072193146, "p05": 0.09093102067708969, "p95": 0.16081735491752625 }, { "label": "current_group_partial", "mean": 0.5200446844100952, "median": 0.5256286263465881, "p05": 0.41434088349342346, "p95": 0.606282114982605 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7546210289001465, "group": 6, "latest_source_mass": 0.3774413764476776, "layer": 24, "max_source_mass": 0.3774413764476776, "mixer_index": 46, "normalized_entropy": 0.9016968382388917, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14427465200424194, "median": 0.14563661813735962, "p05": 0.09999129176139832, "p95": 0.19150450825691223 }, { "label": "completed_group_1", "mean": 0.12224892526865005, "median": 0.1226624846458435, "p05": 0.09991119801998138, "p95": 0.1422707736492157 }, { "label": "completed_group_2", "mean": 0.08193168044090271, "median": 0.08116704970598221, "p05": 0.07012136280536652, "p95": 0.09678678959608078 }, { "label": "completed_group_3", "mean": 0.07876674830913544, "median": 0.0766892209649086, "p05": 0.06478038430213928, "p95": 0.09861677885055542 }, { "label": "completed_group_4", "mean": 0.09007281810045242, "median": 0.08881355822086334, "p05": 0.07590551674365997, "p95": 0.11008355021476746 }, { "label": "completed_group_5", "mean": 0.10526381433010101, "median": 0.10412947833538055, "p05": 0.08725899457931519, "p95": 0.12721270322799683 }, { "label": "current_group_partial", "mean": 0.3774413764476776, "median": 0.37542667984962463, "p05": 0.3145911991596222, "p95": 0.44844120740890503 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5284571647644043, "group": 6, "latest_source_mass": 0.5105152726173401, "layer": 24, "max_source_mass": 0.5105152726173401, "mixer_index": 47, "normalized_entropy": 0.7854716033555963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10831617563962936, "median": 0.09708426892757416, "p05": 0.04612221196293831, "p95": 0.24017156660556793 }, { "label": "completed_group_1", "mean": 0.08754566311836243, "median": 0.08632134646177292, "p05": 0.06808170676231384, "p95": 0.10904698818922043 }, { "label": "completed_group_2", "mean": 0.06345342099666595, "median": 0.06274682283401489, "p05": 0.047288306057453156, "p95": 0.08281171321868896 }, { "label": "completed_group_3", "mean": 0.06688661873340607, "median": 0.06420055776834488, "p05": 0.048897963017225266, "p95": 0.09314233064651489 }, { "label": "completed_group_4", "mean": 0.05242554843425751, "median": 0.051159247756004333, "p05": 0.03922025114297867, "p95": 0.07083941996097565 }, { "label": "completed_group_5", "mean": 0.11085730046033859, "median": 0.10786899924278259, "p05": 0.08087077736854553, "p95": 0.15321578085422516 }, { "label": "current_group_partial", "mean": 0.5105152726173401, "median": 0.5166662931442261, "p05": 0.39341118931770325, "p95": 0.6058120131492615 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8309729099273682, "group": 7, "latest_source_mass": 0.31088557839393616, "layer": 25, "max_source_mass": 0.31088557839393616, "mixer_index": 48, "normalized_entropy": 0.9409339433355933, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1468513309955597, "median": 0.14672473073005676, "p05": 0.08069700002670288, "p95": 0.2256799042224884 }, { "label": "completed_group_1", "mean": 0.13658970594406128, "median": 0.13762319087982178, "p05": 0.11331360042095184, "p95": 0.1581028401851654 }, { "label": "completed_group_2", "mean": 0.09501184523105621, "median": 0.09421288967132568, "p05": 0.07920350134372711, "p95": 0.11351083964109421 }, { "label": "completed_group_3", "mean": 0.0928751602768898, "median": 0.09060071408748627, "p05": 0.07744356989860535, "p95": 0.11498282104730606 }, { "label": "completed_group_4", "mean": 0.10057065635919571, "median": 0.09926527738571167, "p05": 0.08480366319417953, "p95": 0.12013090401887894 }, { "label": "completed_group_5", "mean": 0.11721574515104294, "median": 0.1158604770898819, "p05": 0.09652461111545563, "p95": 0.14446426928043365 }, { "label": "completed_group_6", "mean": 0.31088557839393616, "median": 0.3095095753669739, "p05": 0.24580135941505432, "p95": 0.3798484206199646 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6937754154205322, "group": 7, "latest_source_mass": 0.26301223039627075, "layer": 25, "max_source_mass": 0.3669250011444092, "mixer_index": 49, "normalized_entropy": 0.8145337974022819, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09603364765644073, "median": 0.08905445039272308, "p05": 0.058942534029483795, "p95": 0.15364733338356018 }, { "label": "completed_group_1", "mean": 0.06767556071281433, "median": 0.06639593839645386, "p05": 0.04534030333161354, "p95": 0.09348243474960327 }, { "label": "completed_group_2", "mean": 0.04511266574263573, "median": 0.044085465371608734, "p05": 0.0323045440018177, "p95": 0.060699306428432465 }, { "label": "completed_group_3", "mean": 0.04789547994732857, "median": 0.04490892216563225, "p05": 0.03212590143084526, "p95": 0.07155333459377289 }, { "label": "completed_group_4", "mean": 0.03816477954387665, "median": 0.036960676312446594, "p05": 0.028077732771635056, "p95": 0.05199173092842102 }, { "label": "completed_group_5", "mean": 0.07518067210912704, "median": 0.07277381420135498, "p05": 0.0526883490383625, "p95": 0.10617287456989288 }, { "label": "completed_group_6", "mean": 0.3669250011444092, "median": 0.36637505888938904, "p05": 0.23747950792312622, "p95": 0.49873870611190796 }, { "label": "current_group_partial", "mean": 0.26301223039627075, "median": 0.26092374324798584, "p05": 0.11381205916404724, "p95": 0.4189145267009735 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9070472717285156, "group": 7, "latest_source_mass": 0.2382715940475464, "layer": 26, "max_source_mass": 0.24526362121105194, "mixer_index": 50, "normalized_entropy": 0.9170958805545191, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10919073224067688, "median": 0.11166982352733612, "p05": 0.04006436467170715, "p95": 0.17950788140296936 }, { "label": "completed_group_1", "mean": 0.11743451654911041, "median": 0.11847291141748428, "p05": 0.078579381108284, "p95": 0.15376348793506622 }, { "label": "completed_group_2", "mean": 0.07601121068000793, "median": 0.07483125478029251, "p05": 0.06124042719602585, "p95": 0.09465125948190689 }, { "label": "completed_group_3", "mean": 0.06814020872116089, "median": 0.06591488420963287, "p05": 0.052837856113910675, "p95": 0.08991338312625885 }, { "label": "completed_group_4", "mean": 0.0658918246626854, "median": 0.06464026868343353, "p05": 0.051906269043684006, "p95": 0.08394908905029297 }, { "label": "completed_group_5", "mean": 0.07979628443717957, "median": 0.07902438938617706, "p05": 0.06372269243001938, "p95": 0.0988437831401825 }, { "label": "completed_group_6", "mean": 0.24526362121105194, "median": 0.24634721875190735, "p05": 0.17033326625823975, "p95": 0.3122789263725281 }, { "label": "current_group_partial", "mean": 0.2382715940475464, "median": 0.22752389311790466, "p05": 0.1175542026758194, "p95": 0.39684510231018066 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.776512861251831, "group": 7, "latest_source_mass": 0.23505288362503052, "layer": 26, "max_source_mass": 0.33455103635787964, "mixer_index": 51, "normalized_entropy": 0.8543220983344934, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08731269836425781, "median": 0.07214296609163284, "p05": 0.03820698708295822, "p95": 0.2289474606513977 }, { "label": "completed_group_1", "mean": 0.08430957794189453, "median": 0.08319784700870514, "p05": 0.05780494213104248, "p95": 0.11521004140377045 }, { "label": "completed_group_2", "mean": 0.056895703077316284, "median": 0.05547339469194412, "p05": 0.041310664266347885, "p95": 0.07652511447668076 }, { "label": "completed_group_3", "mean": 0.06295455247163773, "median": 0.0598134770989418, "p05": 0.04222813621163368, "p95": 0.09349943697452545 }, { "label": "completed_group_4", "mean": 0.04594648629426956, "median": 0.0447731539607048, "p05": 0.034647077322006226, "p95": 0.06127878278493881 }, { "label": "completed_group_5", "mean": 0.09297706186771393, "median": 0.09176799654960632, "p05": 0.0599244087934494, "p95": 0.13207381963729858 }, { "label": "completed_group_6", "mean": 0.33455103635787964, "median": 0.3391231298446655, "p05": 0.2096378207206726, "p95": 0.4444153904914856 }, { "label": "current_group_partial", "mean": 0.23505288362503052, "median": 0.22023122012615204, "p05": 0.0984557569026947, "p95": 0.41664981842041016 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9573358297348022, "group": 7, "latest_source_mass": 0.2227109968662262, "layer": 27, "max_source_mass": 0.22605091333389282, "mixer_index": 52, "normalized_entropy": 0.9412795649708946, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11900999397039413, "median": 0.12137913703918457, "p05": 0.07443303614854813, "p95": 0.16705159842967987 }, { "label": "completed_group_1", "mean": 0.11752338707447052, "median": 0.1196502298116684, "p05": 0.0804249495267868, "p95": 0.14694978296756744 }, { "label": "completed_group_2", "mean": 0.07979564368724823, "median": 0.07969067245721817, "p05": 0.0651809498667717, "p95": 0.09441538900136948 }, { "label": "completed_group_3", "mean": 0.07266241312026978, "median": 0.07125541567802429, "p05": 0.05800755321979523, "p95": 0.09137322753667831 }, { "label": "completed_group_4", "mean": 0.07412993162870407, "median": 0.07290434837341309, "p05": 0.05999685451388359, "p95": 0.09376152604818344 }, { "label": "completed_group_5", "mean": 0.08811671286821365, "median": 0.087425097823143, "p05": 0.07325023412704468, "p95": 0.10507422685623169 }, { "label": "completed_group_6", "mean": 0.22605091333389282, "median": 0.22650857269763947, "p05": 0.1749759018421173, "p95": 0.27763867378234863 }, { "label": "current_group_partial", "mean": 0.2227109968662262, "median": 0.2145734429359436, "p05": 0.14350463449954987, "p95": 0.328477680683136 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7231189012527466, "group": 7, "latest_source_mass": 0.2634544372558594, "layer": 27, "max_source_mass": 0.3417898416519165, "mixer_index": 53, "normalized_entropy": 0.8286450312331257, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09091638028621674, "median": 0.07839073240756989, "p05": 0.045465391129255295, "p95": 0.19545325636863708 }, { "label": "completed_group_1", "mean": 0.07795695960521698, "median": 0.07602830231189728, "p05": 0.053584761917591095, "p95": 0.1065598800778389 }, { "label": "completed_group_2", "mean": 0.05235600098967552, "median": 0.05057134851813316, "p05": 0.03418106958270073, "p95": 0.07707518339157104 }, { "label": "completed_group_3", "mean": 0.053361356258392334, "median": 0.05035271495580673, "p05": 0.031062742695212364, "p95": 0.08295466005802155 }, { "label": "completed_group_4", "mean": 0.04162060469388962, "median": 0.040454644709825516, "p05": 0.029850685968995094, "p95": 0.05732571333646774 }, { "label": "completed_group_5", "mean": 0.07854442298412323, "median": 0.07773076742887497, "p05": 0.053876958787441254, "p95": 0.10653360933065414 }, { "label": "completed_group_6", "mean": 0.3417898416519165, "median": 0.348370760679245, "p05": 0.1893925964832306, "p95": 0.4778548777103424 }, { "label": "current_group_partial", "mean": 0.2634544372558594, "median": 0.2390930950641632, "p05": 0.11348576098680496, "p95": 0.4878687262535095 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8967883586883545, "group": 7, "latest_source_mass": 0.23599036037921906, "layer": 28, "max_source_mass": 0.261149525642395, "mixer_index": 54, "normalized_entropy": 0.9121623862318685, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1089041605591774, "median": 0.1106237843632698, "p05": 0.0463758185505867, "p95": 0.1698746681213379 }, { "label": "completed_group_1", "mean": 0.10000172257423401, "median": 0.09870216995477676, "p05": 0.07011638581752777, "p95": 0.13429011404514313 }, { "label": "completed_group_2", "mean": 0.07706406712532043, "median": 0.0757090300321579, "p05": 0.05966329202055931, "p95": 0.09858658909797668 }, { "label": "completed_group_3", "mean": 0.07702124863862991, "median": 0.07400410622358322, "p05": 0.05783015862107277, "p95": 0.1036195158958435 }, { "label": "completed_group_4", "mean": 0.06017591059207916, "median": 0.05848415568470955, "p05": 0.04412911832332611, "p95": 0.08321075141429901 }, { "label": "completed_group_5", "mean": 0.07969298958778381, "median": 0.07816708087921143, "p05": 0.057708315551280975, "p95": 0.10603620111942291 }, { "label": "completed_group_6", "mean": 0.261149525642395, "median": 0.2585980296134949, "p05": 0.1826809048652649, "p95": 0.34350520372390747 }, { "label": "current_group_partial", "mean": 0.23599036037921906, "median": 0.22840067744255066, "p05": 0.1214975118637085, "p95": 0.3747939467430115 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.801592230796814, "group": 7, "latest_source_mass": 0.1870974600315094, "layer": 28, "max_source_mass": 0.3473600149154663, "mixer_index": 55, "normalized_entropy": 0.8663827256915495, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09432460367679596, "median": 0.07062139362096786, "p05": 0.0350821278989315, "p95": 0.24959293007850647 }, { "label": "completed_group_1", "mean": 0.09541776776313782, "median": 0.09502051025629044, "p05": 0.06895694881677628, "p95": 0.12235064059495926 }, { "label": "completed_group_2", "mean": 0.06706254929304123, "median": 0.0657147765159607, "p05": 0.049946803599596024, "p95": 0.08819274604320526 }, { "label": "completed_group_3", "mean": 0.07247316837310791, "median": 0.06966830790042877, "p05": 0.05053037777543068, "p95": 0.09898276627063751 }, { "label": "completed_group_4", "mean": 0.048089414834976196, "median": 0.0466206818819046, "p05": 0.035116635262966156, "p95": 0.06637651473283768 }, { "label": "completed_group_5", "mean": 0.08817502856254578, "median": 0.08746744692325592, "p05": 0.062397219240665436, "p95": 0.11551516503095627 }, { "label": "completed_group_6", "mean": 0.3473600149154663, "median": 0.354450523853302, "p05": 0.20967835187911987, "p95": 0.4590650796890259 }, { "label": "current_group_partial", "mean": 0.1870974600315094, "median": 0.15943653881549835, "p05": 0.072502501308918, "p95": 0.38723331689834595 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.919801950454712, "group": 8, "latest_source_mass": 0.19988907873630524, "layer": 29, "max_source_mass": 0.27338963747024536, "mixer_index": 56, "normalized_entropy": 0.9232295844699908, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10887958109378815, "median": 0.10778149962425232, "p05": 0.06027446314692497, "p95": 0.16913464665412903 }, { "label": "completed_group_1", "mean": 0.10445677489042282, "median": 0.10656317323446274, "p05": 0.06373343616724014, "p95": 0.13975882530212402 }, { "label": "completed_group_2", "mean": 0.0838727131485939, "median": 0.08285491168498993, "p05": 0.06476438045501709, "p95": 0.10624834150075912 }, { "label": "completed_group_3", "mean": 0.0852852314710617, "median": 0.0823126882314682, "p05": 0.06312129646539688, "p95": 0.1139926165342331 }, { "label": "completed_group_4", "mean": 0.05865198373794556, "median": 0.055648073554039, "p05": 0.04130495712161064, "p95": 0.08606290817260742 }, { "label": "completed_group_5", "mean": 0.08557502925395966, "median": 0.08467335999011993, "p05": 0.06446348130702972, "p95": 0.10952071845531464 }, { "label": "completed_group_6", "mean": 0.27338963747024536, "median": 0.2741563320159912, "p05": 0.19311219453811646, "p95": 0.3497011959552765 }, { "label": "completed_group_7", "mean": 0.19988907873630524, "median": 0.1887303590774536, "p05": 0.10821843147277832, "p95": 0.3243493437767029 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.9332969188690186, "group": 8, "latest_source_mass": 0.25055450201034546, "layer": 29, "max_source_mass": 0.2511133551597595, "mixer_index": 57, "normalized_entropy": 0.8798813461356915, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07704077661037445, "median": 0.0657581090927124, "p05": 0.031858570873737335, "p95": 0.16932255029678345 }, { "label": "completed_group_1", "mean": 0.08837630599737167, "median": 0.08667970448732376, "p05": 0.05486329644918442, "p95": 0.12723307311534882 }, { "label": "completed_group_2", "mean": 0.05746292322874069, "median": 0.05622619390487671, "p05": 0.04283035546541214, "p95": 0.0759863406419754 }, { "label": "completed_group_3", "mean": 0.06001049652695656, "median": 0.05710422247648239, "p05": 0.0414297915995121, "p95": 0.08556482195854187 }, { "label": "completed_group_4", "mean": 0.04015304893255234, "median": 0.03914806246757507, "p05": 0.02911726012825966, "p95": 0.05505921691656113 }, { "label": "completed_group_5", "mean": 0.06584024429321289, "median": 0.06438547372817993, "p05": 0.04402649402618408, "p95": 0.09364330768585205 }, { "label": "completed_group_6", "mean": 0.2511133551597595, "median": 0.24885429441928864, "p05": 0.16403883695602417, "p95": 0.34440478682518005 }, { "label": "completed_group_7", "mean": 0.10944834351539612, "median": 0.08987390995025635, "p05": 0.04556909203529358, "p95": 0.22957125306129456 }, { "label": "current_group_partial", "mean": 0.25055450201034546, "median": 0.2451469898223877, "p05": 0.11517379432916641, "p95": 0.4031696617603302 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0115504264831543, "group": 8, "latest_source_mass": 0.16160473227500916, "layer": 30, "max_source_mass": 0.2653427720069885, "mixer_index": 58, "normalized_entropy": 0.9154960522614556, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08485299348831177, "median": 0.07469280064105988, "p05": 0.04144430533051491, "p95": 0.1810980588197708 }, { "label": "completed_group_1", "mean": 0.08635596185922623, "median": 0.08461107313632965, "p05": 0.05276995152235031, "p95": 0.12535785138607025 }, { "label": "completed_group_2", "mean": 0.06204982101917267, "median": 0.061112865805625916, "p05": 0.045534878969192505, "p95": 0.08121293038129807 }, { "label": "completed_group_3", "mean": 0.06870189309120178, "median": 0.06677035242319107, "p05": 0.046723417937755585, "p95": 0.0936926007270813 }, { "label": "completed_group_4", "mean": 0.0476498119533062, "median": 0.042902059853076935, "p05": 0.030372878536581993, "p95": 0.08035364747047424 }, { "label": "completed_group_5", "mean": 0.08640170097351074, "median": 0.08549860864877701, "p05": 0.059603940695524216, "p95": 0.11531224846839905 }, { "label": "completed_group_6", "mean": 0.2653427720069885, "median": 0.26641854643821716, "p05": 0.1776670217514038, "p95": 0.3456951975822449 }, { "label": "completed_group_7", "mean": 0.13704031705856323, "median": 0.13101622462272644, "p05": 0.0725383460521698, "p95": 0.22235709428787231 }, { "label": "current_group_partial", "mean": 0.16160473227500916, "median": 0.15514495968818665, "p05": 0.08618976920843124, "p95": 0.255747526884079 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781036376953125, "group": 8, "latest_source_mass": 0.15038734674453735, "layer": 30, "max_source_mass": 0.29481440782546997, "mixer_index": 59, "normalized_entropy": 0.9002737626817574, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08389372378587723, "median": 0.06660648435354233, "p05": 0.04010086879134178, "p95": 0.22044557332992554 }, { "label": "completed_group_1", "mean": 0.08841055631637573, "median": 0.08784788101911545, "p05": 0.06586075574159622, "p95": 0.11172220855951309 }, { "label": "completed_group_2", "mean": 0.060679107904434204, "median": 0.0598948709666729, "p05": 0.045160289853811264, "p95": 0.07931925356388092 }, { "label": "completed_group_3", "mean": 0.0658775195479393, "median": 0.06333823502063751, "p05": 0.047597575932741165, "p95": 0.09037967771291733 }, { "label": "completed_group_4", "mean": 0.045975662767887115, "median": 0.04452216252684593, "p05": 0.03386770188808441, "p95": 0.06284140050411224 }, { "label": "completed_group_5", "mean": 0.08423998951911926, "median": 0.0840892568230629, "p05": 0.05936872214078903, "p95": 0.1089530736207962 }, { "label": "completed_group_6", "mean": 0.29481440782546997, "median": 0.3008817434310913, "p05": 0.17160090804100037, "p95": 0.396139919757843 }, { "label": "completed_group_7", "mean": 0.12572166323661804, "median": 0.10853725671768188, "p05": 0.06020044535398483, "p95": 0.24606993794441223 }, { "label": "current_group_partial", "mean": 0.15038734674453735, "median": 0.14419245719909668, "p05": 0.07393782585859299, "p95": 0.24871884286403656 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0586161613464355, "group": 8, "latest_source_mass": 0.1407114714384079, "layer": 31, "max_source_mass": 0.22791332006454468, "mixer_index": 60, "normalized_entropy": 0.9369165913127441, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1038442850112915, "median": 0.10749004781246185, "p05": 0.021369464695453644, "p95": 0.1852288693189621 }, { "label": "completed_group_1", "mean": 0.09918913245201111, "median": 0.09865356981754303, "p05": 0.057106852531433105, "p95": 0.1428801417350769 }, { "label": "completed_group_2", "mean": 0.08148117363452911, "median": 0.08018780499696732, "p05": 0.060506708920001984, "p95": 0.10691888630390167 }, { "label": "completed_group_3", "mean": 0.0698266476392746, "median": 0.06695331633090973, "p05": 0.0502583347260952, "p95": 0.09614066034555435 }, { "label": "completed_group_4", "mean": 0.059573933482170105, "median": 0.05744876712560654, "p05": 0.04474463313817978, "p95": 0.08047857880592346 }, { "label": "completed_group_5", "mean": 0.08160553872585297, "median": 0.08077611029148102, "p05": 0.060524459928274155, "p95": 0.10581386834383011 }, { "label": "completed_group_6", "mean": 0.22791332006454468, "median": 0.22841286659240723, "p05": 0.15760843455791473, "p95": 0.29978859424591064 }, { "label": "completed_group_7", "mean": 0.13585451245307922, "median": 0.12730783224105835, "p05": 0.05973683297634125, "p95": 0.2417306751012802 }, { "label": "current_group_partial", "mean": 0.1407114714384079, "median": 0.1365126073360443, "p05": 0.052046023309230804, "p95": 0.2530304193496704 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.022735118865967, "group": 8, "latest_source_mass": 0.14314599335193634, "layer": 31, "max_source_mass": 0.24251294136047363, "mixer_index": 61, "normalized_entropy": 0.9205864251337508, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09707076847553253, "median": 0.09347058087587357, "p05": 0.03571572154760361, "p95": 0.18511834740638733 }, { "label": "completed_group_1", "mean": 0.09551472961902618, "median": 0.0936870127916336, "p05": 0.07090321183204651, "p95": 0.12828916311264038 }, { "label": "completed_group_2", "mean": 0.06701318919658661, "median": 0.06529247015714645, "p05": 0.04918193817138672, "p95": 0.09029978513717651 }, { "label": "completed_group_3", "mean": 0.06734597682952881, "median": 0.06535796821117401, "p05": 0.04815492779016495, "p95": 0.09132430702447891 }, { "label": "completed_group_4", "mean": 0.04480960592627525, "median": 0.04339977353811264, "p05": 0.034005217254161835, "p95": 0.06039253994822502 }, { "label": "completed_group_5", "mean": 0.08329526335000992, "median": 0.08157584071159363, "p05": 0.057632118463516235, "p95": 0.11397058516740799 }, { "label": "completed_group_6", "mean": 0.24251294136047363, "median": 0.24409571290016174, "p05": 0.14804454147815704, "p95": 0.3317493200302124 }, { "label": "completed_group_7", "mean": 0.15929152071475983, "median": 0.14274996519088745, "p05": 0.07706648111343384, "p95": 0.29802852869033813 }, { "label": "current_group_partial", "mean": 0.14314599335193634, "median": 0.13742807507514954, "p05": 0.05530726909637451, "p95": 0.25646957755088806 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9317231178283691, "group": 8, "latest_source_mass": 0.09359145164489746, "layer": 32, "max_source_mass": 0.3483121693134308, "mixer_index": 62, "normalized_entropy": 0.8791650784146389, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.07746735960245132, "median": 0.058669812977313995, "p05": 0.03224737197160721, "p95": 0.21910449862480164 }, { "label": "completed_group_1", "mean": 0.08971680700778961, "median": 0.08852710574865341, "p05": 0.051282018423080444, "p95": 0.13161666691303253 }, { "label": "completed_group_2", "mean": 0.06909166276454926, "median": 0.06691395491361618, "p05": 0.0473506934940815, "p95": 0.09751966595649719 }, { "label": "completed_group_3", "mean": 0.08098623156547546, "median": 0.07793208211660385, "p05": 0.05465787276625633, "p95": 0.11334780603647232 }, { "label": "completed_group_4", "mean": 0.04421798139810562, "median": 0.03908399119973183, "p05": 0.025279562920331955, "p95": 0.07900907099246979 }, { "label": "completed_group_5", "mean": 0.0916394293308258, "median": 0.09000804275274277, "p05": 0.05948498472571373, "p95": 0.1287805736064911 }, { "label": "completed_group_6", "mean": 0.3483121693134308, "median": 0.3540991246700287, "p05": 0.21832552552223206, "p95": 0.45785486698150635 }, { "label": "completed_group_7", "mean": 0.10497690737247467, "median": 0.09170983731746674, "p05": 0.0477774515748024, "p95": 0.20629161596298218 }, { "label": "current_group_partial", "mean": 0.09359145164489746, "median": 0.09055774658918381, "p05": 0.05538146197795868, "p95": 0.14527763426303864 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.053220272064209, "group": 8, "latest_source_mass": 0.10435353964567184, "layer": 32, "max_source_mass": 0.26201331615448, "mixer_index": 63, "normalized_entropy": 0.934460816269135, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09178322553634644, "median": 0.07441723346710205, "p05": 0.05449800193309784, "p95": 0.19793085753917694 }, { "label": "completed_group_1", "mean": 0.10647952556610107, "median": 0.10643037408590317, "p05": 0.08102327585220337, "p95": 0.13105911016464233 }, { "label": "completed_group_2", "mean": 0.07523074746131897, "median": 0.07466189563274384, "p05": 0.0580206960439682, "p95": 0.09448636323213577 }, { "label": "completed_group_3", "mean": 0.08394691348075867, "median": 0.08230198919773102, "p05": 0.06343884021043777, "p95": 0.10998549312353134 }, { "label": "completed_group_4", "mean": 0.04770101234316826, "median": 0.046069372445344925, "p05": 0.036705970764160156, "p95": 0.06487860530614853 }, { "label": "completed_group_5", "mean": 0.09626082330942154, "median": 0.09582744538784027, "p05": 0.072615846991539, "p95": 0.1200246512889862 }, { "label": "completed_group_6", "mean": 0.26201331615448, "median": 0.2656516432762146, "p05": 0.17182716727256775, "p95": 0.3362464904785156 }, { "label": "completed_group_7", "mean": 0.13223090767860413, "median": 0.12176793813705444, "p05": 0.0719129890203476, "p95": 0.22232075035572052 }, { "label": "current_group_partial", "mean": 0.10435353964567184, "median": 0.09856735169887543, "p05": 0.06300172954797745, "p95": 0.16237276792526245 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.843801498413086, "group": null, "latest_source_mass": 0.19098761677742004, "layer": null, "max_source_mass": 0.3250349760055542, "mixer_index": null, "normalized_entropy": 0.8391502249844656, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.11151862144470215, "median": 0.11458685994148254, "p05": 0.04668144881725311, "p95": 0.1599162518978119 }, { "label": "completed_group_1", "mean": 0.061874739825725555, "median": 0.06061127781867981, "p05": 0.04380229860544205, "p95": 0.08225110918283463 }, { "label": "completed_group_2", "mean": 0.05308101326227188, "median": 0.05128483101725578, "p05": 0.03527887910604477, "p95": 0.07726569473743439 }, { "label": "completed_group_3", "mean": 0.040848225355148315, "median": 0.038592323660850525, "p05": 0.024852607399225235, "p95": 0.06626670062541962 }, { "label": "completed_group_4", "mean": 0.042743273079395294, "median": 0.04188907518982887, "p05": 0.03039409965276718, "p95": 0.0579918697476387 }, { "label": "completed_group_5", "mean": 0.053944505751132965, "median": 0.05261969938874245, "p05": 0.037368807941675186, "p95": 0.07573261857032776 }, { "label": "completed_group_6", "mean": 0.1199670284986496, "median": 0.10659539699554443, "p05": 0.05845983326435089, "p95": 0.2257011979818344 }, { "label": "completed_group_7", "mean": 0.3250349760055542, "median": 0.3179755210876465, "p05": 0.12162069231271744, "p95": 0.540380597114563 }, { "label": "completed_group_8", "mean": 0.19098761677742004, "median": 0.1842590719461441, "p05": 0.022588828578591347, "p95": 0.4012773036956787 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "b1f88fe26521a0d9ae8700286d108b588142750400633243896bf96a8e44a5fb", "optimizer_state_before": "b1f88fe26521a0d9ae8700286d108b588142750400633243896bf96a8e44a5fb", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "3bc7ac091ccda73a60054802f3ec6d7bc68d1af6b8f7940e83f1843e7ab472be", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.051234737038612366, 0.11323792487382889, 0.05108218640089035, 0.03771067038178444, 0.08815877884626389, 0.03706873208284378, 0.0438041090965271, 0.030363978818058968, 0.03348636254668236, 0.027010800316929817, 0.032826121896505356, 0.031702958047389984, 0.038218315690755844, 0.03341353312134743, 0.04180334508419037, 0.2476748526096344, 0.05078785493969917, 0.0355411171913147, 0.052597854286432266, 0.06600882858037949, 0.050489071756601334, 0.022040290758013725, 0.028325051069259644, 0.02171224355697632, 0.024430278688669205, 0.015834001824259758, 0.018936052918434143, 0.02182060107588768, 0.030554499477148056, 0.016542606055736542, 0.015565099194645882, 0.013472299091517925 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1110142281722801e-05, "first_to_last_ratio": 1.1615894542734118, "imbalance_abs_log_ratio": 0.1497892864258095, "last_quartile_mean": 9.56460326051456e-06, "mean": 1.3397138474147141e-05, "normalized": [ 0.9679183570279057, 1.0382751270469845, 0.9205849321245912, 0.8794696386057994, 0.7953559954151083, 0.7288729389713584, 0.666621226062364, 0.6372392908707228, 0.6772537249699894, 0.6436456735607675, 0.6038963391182133, 0.5706397399299389, 0.5513013139860934, 0.5295867329502082, 0.5131681370281796, 0.5688577678712857, 1.1361761926655034, 1.1401114120766789, 1.0979237717774961, 1.045258619130553, 3.411268167406657, 2.8827386084395865, 2.3438877456548695, 1.9385179883302053, 1.2367088391016392, 1.0924371253612384, 0.9526017067211789, 0.8037245228882399, 0.49817350982557956, 0.40700227563318875, 0.3703725294044812, 0.35041005004339426 ], "peak_layer": 21, "peak_normalized": 3.411268167406657, "population_cv": 0.694220928383246, "rest_mean": 1.0016533653672215e-05, "spike_contrast": 3.160015789804438, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.165240450471174e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.2967336260771845e-05, 1.3909915651311167e-05, 1.2333203812886495e-05, 1.1782376532210037e-05, 1.0655494406819344e-05, 9.764811693457887e-06, 8.930816875363234e-06, 8.537183020962402e-06, 9.07326193555491e-06, 8.62301021697931e-06, 8.090482879197225e-06, 7.644939614692703e-06, 7.385860044450965e-06, 7.0949467954051215e-06, 6.874984592286637e-06, 7.621066288265865e-06, 1.522150978416903e-05, 1.52742304635467e-05, 1.4709036804561038e-05, 1.4003474461787846e-05, 4.5701232011197135e-05, 3.862044832203537e-05, 3.140138869639486e-05, 2.5970593924284913e-05, 1.6568359569646418e-05, 1.463553144276375e-05, 1.2762136975652538e-05, 1.0767608728201594e-05, 6.674099495285191e-06, 5.452665845950833e-06, 4.961932063451968e-06, 4.694491963164182e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.520096763371839e-05, "first_to_last_ratio": 1.1880767713048908, "imbalance_abs_log_ratio": 0.17233584116303574, "last_quartile_mean": 3.804549396591028e-05, "mean": 5.4024936389396316e-05, "normalized": [ 0.9708707190563224, 1.0451943937367467, 0.9250577208903767, 0.8842743984856529, 0.8031024942014594, 0.7427444322392254, 0.6768740767497347, 0.6452305260972615, 0.6822213900165217, 0.6476553992317551, 0.6085575017002121, 0.5751826010571539, 0.5529292217889952, 0.5299244079385589, 0.5136466137046081, 0.568975504947234, 1.1417881140585813, 1.159000740319458, 1.1039052119717943, 1.0463955173718469, 3.422947145673778, 2.8780196348647618, 2.3242594176046136, 1.9174749293029072, 1.2126088201065046, 1.0731537668257296, 0.9360922740994, 0.7921003983334755, 0.4948022653288053, 0.40618173797049695, 0.3695278378158856, 0.34930078651014324 ], "peak_layer": 21, "peak_normalized": 3.422947145673778, "population_cv": 0.6923039078369757, "rest_mean": 4.050807749283397e-05, "spike_contrast": 3.1355715277601766, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012701597443083302, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 5.2451228839345276e-05, 5.6466560636181384e-05, 4.9976184527622536e-05, 4.777286812895909e-05, 4.338756116339937e-05, 4.012672070530243e-05, 3.6568078940035775e-05, 3.485853812890127e-05, 3.685696719912812e-05, 3.4989541745744646e-05, 3.28772803186439e-05, 3.107420343440026e-05, 2.9871966034988873e-05, 2.862913243006915e-05, 2.7749725632020272e-05, 3.073886546189897e-05, 6.168503023218364e-05, 6.261494127102196e-05, 5.963840885669924e-05, 5.653145126416348e-05, 0.00018492450180929154, 0.00015548482770100236, 0.00012556796718854457, 0.00010359146108385175, 6.551111437147483e-05, 5.797706398880109e-05, 5.0572325562825426e-05, 4.2793173633981496e-05, 2.6731660909717903e-05, 2.194394255639054e-05, 1.9963717932114378e-05, 1.887095277197659e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04008466936647892, "first_to_last_ratio": 1.1880767439372768, "imbalance_abs_log_ratio": 0.17233581812781198, "last_quartile_mean": 0.03373912465758622, "mean": 0.04790985374711454, "normalized": [ 0.970870766181603, 1.0451943834429158, 0.9250577200396625, 0.8842743365685358, 0.8031024935917312, 0.7427444421233665, 0.6768740172187802, 0.645230491498922, 0.682221391904326, 0.6476553588595259, 0.6085574586207388, 0.5751825734843286, 0.552929202866552, 0.5299244347807472, 0.5136466315472489, 0.5689755646385108, 1.1417881666708014, 1.1590007554748112, 1.1039052466293215, 1.0463955619068368, 3.422947136129388, 2.878019556230048, 2.324259489013662, 1.9174748970647042, 1.212608709608152, 1.0731538183017837, 0.9360922641989038, 0.7921004515426608, 0.49480228907579926, 0.40618174778590915, 0.3695278429438553, 0.3493008000558679 ], "peak_layer": 21, "peak_normalized": 3.422947136129388, "population_cv": 0.692303902772671, "rest_mean": 0.0359229684151985, "spike_contrast": 3.135571460508958, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11263903453946114, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.046514276415109634, 0.05007511004805565, 0.044319380074739456, 0.04236545413732529, 0.038476523011922836, 0.03558477759361267, 0.032428935170173645, 0.03091289848089218, 0.03268512710928917, 0.031029073521494865, 0.029155898839235306, 0.02755691297352314, 0.02649075724184513, 0.025388602167367935, 0.024608734995126724, 0.02725953608751297, 0.05470290407538414, 0.05552755668759346, 0.0528879389166832, 0.05013265833258629, 0.16399289667606354, 0.13788549602031708, 0.11135493218898773, 0.09186594188213348, 0.05809590592980385, 0.05141464248299599, 0.04484804347157478, 0.037949416786432266, 0.02370590530335903, 0.019460108131170273, 0.01770402491092682, 0.016734950244426727 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8672176622658299e-06, "first_to_last_ratio": 0.7390774425516208, "imbalance_abs_log_ratio": 0.3023525698145062, "last_quartile_mean": 2.526416793102726e-06, "mean": 2.8231130428935103e-06, "normalized": [ 1.0710459542190351, 0.7807005916993321, 0.7446702739163947, 0.7228305497755413, 0.5071025265492799, 0.49702926125096536, 0.48433377289158164, 0.4835173786395229, 0.5122484319159906, 0.5011695032399489, 0.4949613551079442, 0.4928871668407519, 0.4875154271640947, 0.48919988211251697, 0.49217873653037764, 0.5089309468432903, 0.8206332361532853, 0.8408570852012511, 0.8609730105775593, 0.8854026612180381, 3.5996391090811306, 3.2819833392882214, 2.8213892706690173, 2.4595638858289472, 1.4921873165629496, 1.3536774320369742, 1.2002934104350649, 1.0429420663389846, 0.6391162919820471, 0.5173593426294659, 0.47263883361652936, 0.44102194968396574 ], "peak_layer": 21, "peak_normalized": 3.5996391090811306, "population_cv": 0.8312576809575059, "rest_mean": 1.9181732619068093e-06, "spike_contrast": 4.019338614155003, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.709787860221695e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.0236838028940838e-06, 2.2040060230210656e-06, 2.1022883629484568e-06, 2.0406323528732173e-06, 1.4316077567855245e-06, 1.4031697901373263e-06, 1.3673289913640474e-06, 1.365024218102917e-06, 1.4461352293437812e-06, 1.4148581612971611e-06, 1.3973318573334836e-06, 1.3914761893829564e-06, 1.3763111610387568e-06, 1.3810665677738143e-06, 1.3894762105337577e-06, 1.4367695939654368e-06, 2.31674039241625e-06, 2.373834604441072e-06, 2.4306241357408e-06, 2.4995918010972673e-06, 1.0162188118556514e-05, 9.265409971703775e-06, 7.965100849105511e-06, 6.943626885913545e-06, 4.21261347582913e-06, 3.8215844142541755e-06, 3.388563982298365e-06, 2.944343350463896e-06, 1.8042975398202543e-06, 1.4605639080400579e-06, 1.3343128557607997e-06, 1.245054818355129e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.5158251421307796e-05, "first_to_last_ratio": 1.372951179293099, "imbalance_abs_log_ratio": 0.3169625684647144, "last_quartile_mean": 2.5607794327697775e-05, "mean": 3.930288400511017e-05, "normalized": [ 1.0457352487354723, 1.1170219051554786, 0.9973660844453545, 0.9523381538668013, 0.848321809820657, 0.7912232478279534, 0.7196571910711226, 0.6847071570545322, 0.7251137125737372, 0.6895283748593514, 0.6494373653889238, 0.6137885297016834, 0.6090137783434721, 0.5840034448633752, 0.5660471252539855, 0.6192854547978126, 1.2024011313555776, 1.209946097482875, 1.1323117727181873, 1.0603690322871115, 3.309119658074755, 2.746842516693067, 2.1557717289384435, 1.7582494269549842, 1.1449656413436453, 0.9974022764386288, 0.8715835424185431, 0.7496749321117195, 0.4337889933103568, 0.3592093283156041, 0.33316446845998277, 0.32261086933680555 ], "peak_layer": 21, "peak_normalized": 3.309119658074755, "population_cv": 0.6495478333321385, "rest_mean": 3.0401582888670317e-05, "spike_contrast": 2.8738605602817255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.736991003388539e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 4.11004111811053e-05, 4.390218236949295e-05, 3.9199363527586684e-05, 3.7429635995067656e-05, 3.3341493690386415e-05, 3.109735553152859e-05, 2.828460310411174e-05, 2.6910965971183032e-05, 2.849906013580039e-05, 2.710045373532921e-05, 2.5524761440465227e-05, 2.412365938653238e-05, 2.3935997887747362e-05, 2.295301965204999e-05, 2.224728450528346e-05, 2.433970439597033e-05, 4.72578321932815e-05, 4.755437112180516e-05, 4.4503118260763586e-05, 4.167556107859127e-05, 0.00013005794608034194, 0.00010795883281389251, 8.472804620396346e-05, 6.910427327966318e-05, 4.5000451791565865e-05, 3.9200785977300256e-05, 3.425574686843902e-05, 2.9464386898325756e-05, 1.7049158486770466e-05, 1.4117962564341724e-05, 1.309432445850689e-05, 1.2679537576332223e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.67776620148652e-05, "first_to_last_ratio": 1.489713992627352, "imbalance_abs_log_ratio": 0.39858415027555666, "last_quartile_mean": 1.7975035575545917e-05, "mean": 2.834268283891106e-05, "normalized": [ 1.106950131948834, 1.1808729223414434, 1.058395413505648, 1.0131391464097315, 0.8887556018396708, 0.8336258013743472, 0.7556781237788452, 0.7208403934347858, 0.7653210639837278, 0.7236752838504213, 0.6826925954443256, 0.6471110368851702, 0.6603765937793802, 0.6314762042173024, 0.6140873745590488, 0.6701983351410581, 1.245239798504983, 1.2536001966002472, 1.1579582641113197, 1.0597246133471363, 3.138008050522486, 2.5781227127810102, 1.9478107697932208, 1.59270961868747, 1.1031600095096015, 0.9421470006775309, 0.8542088565799294, 0.7657556804352058, 0.41919703377159895, 0.3415465986591183, 0.32698829528201995, 0.3206264782433822 ], "peak_layer": 21, "peak_normalized": 3.138008050522486, "population_cv": 0.5950468550457515, "rest_mean": 2.2716333660355303e-05, "spike_contrast": 2.585142887982815, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.8724968403112145e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 3.137393650831655e-05, 3.346910671098158e-05, 2.99977655231487e-05, 2.8715081498376094e-05, 2.518971814424731e-05, 2.362719169468619e-05, 2.1417945390567183e-05, 2.0430550648598e-05, 2.1691252186428756e-05, 2.0510899048531428e-05, 1.9349339709151536e-05, 1.8340862879995257e-05, 1.8716844351729378e-05, 1.7897729776450433e-05, 1.74048836925067e-05, 1.899521885206923e-05, 3.5293436667416245e-05, 3.5530392779037356e-05, 3.2819643820403144e-05, 3.003543861268554e-05, 8.893956692190841e-05, 7.307091436814517e-05, 5.520618287846446e-05, 4.514166357694194e-05, 3.126651427010074e-05, 2.670297362783458e-05, 2.4210570700233802e-05, 2.1703570382669568e-05, 1.1881168575200718e-05, 9.680346920504235e-06, 9.267725545214489e-06, 9.087414582609199e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.124184998654528e-05, "first_to_last_ratio": 1.1295643250458662, "imbalance_abs_log_ratio": 0.12183200532280404, "last_quartile_mean": 8.07761434771237e-05, "mean": 0.0001131998267283052, "normalized": [ 0.9391291478086077, 0.9994540633197246, 0.8981563809156281, 0.8600090937959485, 0.7637374860603604, 0.7087527588156981, 0.6498801446644327, 0.6290780728741097, 0.6694944081285747, 0.6399004336382168, 0.5999417388211115, 0.5645228757078199, 0.523886204579862, 0.5044184246208804, 0.48957885496141057, 0.5347736119904029, 1.1366324271717236, 1.1355437314234091, 1.0782169392550793, 1.0313830324460944, 3.519062137269261, 2.963582030922574, 2.41128509662923, 2.041010789775174, 1.2315559238773703, 1.0997748890711536, 0.9743076211416006, 0.8521146034254664, 0.47220702964033107, 0.3868732211836673, 0.3540802629057541, 0.3376565631593225 ], "peak_layer": 21, "peak_normalized": 3.519062137269261, "population_cv": 0.7265013683583667, "rest_mean": 8.315367476488636e-05, "spike_contrast": 3.3125300608732915, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002754490473307669, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.0001063092568074353, 0.0001131380267906934, 0.00010167114669457078, 9.735288040246814e-05, 8.645495108794421e-05, 8.023068949114531e-05, 7.356631977017969e-05, 7.121152884792536e-05, 7.57866509957239e-05, 7.24366182112135e-05, 6.791330088162795e-05, 6.390389171428978e-05, 5.9303827583789825e-05, 5.7100078265648335e-05, 5.542024155147374e-05, 6.0536280216183513e-05, 0.0001286665938096121, 0.00012854335363954306, 0.00012205397069919854, 0.00011675238056341186, 0.00039835722418501973, 0.0003354769723955542, 0.00027295705513097346, 0.00023104206775315106, 0.00013941191718913615, 0.00012449432688299567, 0.00011029145389329642, 9.64592254604213e-05, 5.3453753935173154e-05, 4.3793981603812426e-05, 4.008182440884411e-05, 3.822266444331035e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b5dfb9be1f653e88219f0cf5f3837c21e4150bbb11f0520e3acfd9e5ac781830", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.07594770938158035, 0.04037529230117798, 0.04642988368868828, 0.02321692928671837, 0.015561277978122234, 0.01583549566566944, 0.020819898694753647, 0.027020027860999107, 0.0216005090624094, 0.024715786799788475, 0.03509491682052612, 0.05490833893418312, 0.037195492535829544, 0.042240556329488754, 0.04918703809380531, 0.04376247525215149, 0.02373618632555008, 0.01774558797478676, 0.022557877004146576, 0.03756217285990715, 0.03271714597940445, 0.03932206705212593, 0.04476166144013405, 0.049177221953868866, 0.0453680194914341, 0.0437605082988739, 0.052028607577085495, 0.060354240238666534, 0.06733348965644836, 0.07210637629032135, 0.09804103523492813, 0.1243291050195694 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0990880355166155e-05, "first_to_last_ratio": 1.2944234394570906, "imbalance_abs_log_ratio": 0.2580653755149735, "last_quartile_mean": 8.49094664090444e-06, "mean": 1.235348285888449e-05, "normalized": [ 1.0282294525621116, 1.1178781387751966, 0.9935960877446004, 0.9494515014869882, 0.8590253613234318, 0.7848034670921512, 0.7091367306470445, 0.6754706186705651, 0.7068887402186641, 0.6749526105270502, 0.6307964655317274, 0.6026934930305775, 0.5785908008702298, 0.5570004776549429, 0.5565634542979354, 0.6239921973919045, 1.2244474917910158, 1.2505131815537707, 1.1714527408188027, 1.1440936503929904, 3.1783601472255105, 2.5998507728769873, 2.116333755445443, 1.7672210038875287, 1.1859194213452948, 1.0362571799584626, 0.8733376553978273, 0.8036621734585419, 0.442583220280086, 0.40164066204717863, 0.38000807949784143, 0.3752492661975972 ], "peak_layer": 21, "peak_normalized": 3.1783601472255105, "population_cv": 0.6201763550129065, "rest_mean": 9.677954056786365e-06, "spike_contrast": 2.769318621782541, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6801338390214368e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 1.2702214917226229e-05, 1.3809688425681088e-05, 1.2274372238607612e-05, 1.1729032848961651e-05, 1.061195507645607e-05, 9.695056178316008e-06, 8.760308446653653e-06, 8.344414709426928e-06, 8.732537935429718e-06, 8.338015504705254e-06, 7.792533324391115e-06, 7.445363735314459e-06, 7.147611540858634e-06, 6.88089585310081e-06, 6.875497092551086e-06, 7.70847691455856e-06, 1.512619110144442e-05, 1.5448193153133616e-05, 1.4471521353698336e-05, 1.4133541299088392e-05, 3.926381759811193e-05, 3.2117211958393455e-05, 2.6144092771573924e-05, 2.1831334379385225e-05, 1.4650235243607312e-05, 1.2801385310012847e-05, 1.0788761755975429e-05, 9.928026884153951e-06, 5.467444225359941e-06, 4.96166103403084e-06, 4.6944232963141985e-06, 4.635635377781e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.471112742976402e-05, "first_to_last_ratio": 1.3207778481961452, "imbalance_abs_log_ratio": 0.2782208419780995, "last_quartile_mean": 3.385211789463938e-05, "mean": 4.9810070152034314e-05, "normalized": [ 1.0308939392414505, 1.1248976892653257, 0.9983120541152328, 0.9550492360145139, 0.8676012153591989, 0.7999351969960798, 0.720536877837289, 0.6838321262895888, 0.712031571445422, 0.6791934724729616, 0.6349550259349246, 0.6062750337433573, 0.5788419659788087, 0.5569013182098084, 0.5581310424199538, 0.6247833800612929, 1.2301955910359532, 1.2710297092994107, 1.176719130778367, 1.1460218197327094, 3.189763931827885, 2.58430753345956, 2.094796372809775, 1.7380029619955741, 1.165603481494619, 1.0196500369605523, 0.8607727657791588, 0.7949214981109073, 0.4416983921863331, 0.40097204003750814, 0.37888754586879986, 0.37448604323767737 ], "peak_layer": 21, "peak_normalized": 3.189763931827885, "population_cv": 0.6175106053465529, "rest_mean": 3.9160909081046055e-05, "spike_contrast": 2.7403740733718513, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010731553993537091, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 5.1348899432923645e-05, 5.6031232816167176e-05, 4.9725993449101225e-05, 4.757106944452971e-05, 4.3215277401031926e-05, 3.9844828279456124e-05, 3.5889992432203144e-05, 3.4061726182699203e-05, 3.54663425241597e-05, 3.3830674510682e-05, 3.162715438520536e-05, 3.0198601962183602e-05, 2.883215893234592e-05, 2.7739293727790937e-05, 2.780054637696594e-05, 3.112050399067812e-05, 6.127612869022414e-05, 6.331007898552343e-05, 5.861246245331131e-05, 5.7083427236648276e-05, 0.00015888236521277577, 0.00012872453953605145, 0.00010434195428388193, 8.657004946144298e-05, 5.805879118270241e-05, 5.078883987152949e-05, 4.28751518484205e-05, 3.9595095586264506e-05, 2.200102790084202e-05, 1.997244544327259e-05, 1.887241523945704e-05, 1.8653176084626466e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0396502751391381, "first_to_last_ratio": 1.3207778547199724, "imbalance_abs_log_ratio": 0.27822084691748217, "last_quartile_mean": 0.030020396690815687, "mean": 0.044172068301122636, "normalized": [ 1.0308939792489324, 1.1248976196994809, 0.9983120766392304, 0.9550492033649147, 0.8676012439568811, 0.7999351931463972, 0.7205369249138741, 0.6838321362546834, 0.7120315669729911, 0.6791934519566257, 0.6349550233781368, 0.6062750468095437, 0.5788419762424374, 0.5569013220973693, 0.5581310235791198, 0.6247833992671911, 1.2301955213417186, 1.2710297742491206, 1.1767191511915103, 1.1460218181799973, 3.189763837856072, 2.58430760811575, 2.094796347658092, 1.7380029451802739, 1.165603513395908, 1.0196500665782366, 0.860772736339134, 0.7949215093532017, 0.44169838206421963, 0.40097203691663513, 0.37888754792641754, 0.3744860161259034 ], "peak_layer": 21, "peak_normalized": 3.189763837856072, "population_cv": 0.6175105993619994, "rest_mean": 0.03472828582205154, "spike_contrast": 2.740374062104645, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09516849368810654, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 0.045536719262599945, 0.04968905448913574, 0.04409750923514366, 0.04218649864196777, 0.038323741406202316, 0.035334791988134384, 0.03182760626077652, 0.030206279829144478, 0.03145190700888634, 0.030001379549503326, 0.02804727666079998, 0.026780422776937485, 0.02556864731013775, 0.024599483236670494, 0.024653801694512367, 0.027597974985837936, 0.05434028059244156, 0.05614401400089264, 0.05197811871767044, 0.05062215402722359, 0.1408984661102295, 0.11415421217679977, 0.0925314873456955, 0.07677118480205536, 0.051487118005752563, 0.04504005238413811, 0.03802211210131645, 0.03511332720518112, 0.019510731101036072, 0.017711764201521873, 0.016736246645450592, 0.016541821882128716 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8471863825197943e-06, "first_to_last_ratio": 0.8244526663369597, "imbalance_abs_log_ratio": 0.19303554756074884, "last_quartile_mean": 2.240500222683295e-06, "mean": 2.553070206801067e-06, "normalized": [ 1.146978253733447, 0.8593901687387808, 0.8257545739301104, 0.8007460276482833, 0.5570271643774695, 0.5465938673635234, 0.5274890368921664, 0.5241465220341085, 0.5489616323415754, 0.5406808894253674, 0.5308783967421735, 0.5296171888458094, 0.5282959997667881, 0.528928139981341, 0.5448236432095829, 0.5665606991220963, 0.8991729664453528, 0.9130923412964861, 0.9272977733988372, 0.9661483913891897, 3.350792350323582, 2.970835292391852, 2.5766840237816986, 2.2685371323868515, 1.4765630521283415, 1.319483155407689, 1.1214078347249905, 1.0614105324031717, 0.5682912920566121, 0.5158443159181672, 0.4828769301866722, 0.4746904116078829 ], "peak_layer": 21, "peak_normalized": 3.350792350323582, "population_cv": 0.7395032169136886, "rest_mean": 1.8303232780925572e-06, "spike_contrast": 3.5271930915695604, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.455903621827019e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.928316007455578e-06, 2.194083435824723e-06, 2.108209400830674e-06, 2.0443608264031354e-06, 1.422129457750998e-06, 1.3954925179859856e-06, 1.346716544503579e-06, 1.3381828694036813e-06, 1.401537588208157e-06, 1.3803962701786077e-06, 1.3553698181567597e-06, 1.3521498658519704e-06, 1.34877677737677e-06, 1.350390675725066e-06, 1.3909730114392005e-06, 1.4464692412730074e-06, 2.2956517113925656e-06, 2.33118885262229e-06, 2.3674563180975383e-06, 2.4666446734045167e-06, 8.55480811878806e-06, 7.584751074318774e-06, 6.578455213457346e-06, 5.791734565718798e-06, 3.769769136852119e-06, 3.3687331324472325e-06, 2.863032932509668e-06, 2.709855607463396e-06, 1.4508875665342202e-06, 1.31698675431835e-06, 1.2328187040111516e-06, 1.211917947330221e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.482982242530852e-05, "first_to_last_ratio": 1.509442359779984, "imbalance_abs_log_ratio": 0.41174028446052374, "last_quartile_mean": 2.307462898443191e-05, "mean": 3.667055275968778e-05, "normalized": [ 1.0984852795741773, 1.1887269251256418, 1.0648771915091166, 1.0174128492633152, 0.9061095386257815, 0.8430641277424028, 0.7598659375227713, 0.7198868197780367, 0.7526461911930203, 0.7186513938933602, 0.6733570083146694, 0.6414550341700876, 0.6322836859770623, 0.6081147060393997, 0.6055693001423398, 0.6699313875166385, 1.2812295994523142, 1.3084849615457006, 1.195994737290222, 1.1475500305866695, 3.1043926075673, 2.476737924383059, 1.9537955304358658, 1.5974462445862758, 1.0791314645326522, 0.9527353825517174, 0.812033687878467, 0.7520888457646006, 0.38753562965456556, 0.35900639892934844, 0.3463472518949916, 0.34505232655842927 ], "peak_layer": 21, "peak_normalized": 3.1043926075673, "population_cv": 0.586465117433983, "rest_mean": 2.9592451870417722e-05, "spike_contrast": 2.5307905505664654, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.489229756174609e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 4.028206240036525e-05, 4.3591273424681276e-05, 3.904963523382321e-05, 3.730909156729467e-05, 3.3227537642233074e-05, 3.091562757617794e-05, 2.7864703952218406e-05, 2.639864760567434e-05, 2.759995186352171e-05, 2.6353343855589628e-05, 2.4692373699508607e-05, 2.352251067350153e-05, 2.3186192265711725e-05, 2.229990241175983e-05, 2.2206560970516875e-05, 2.456675429129973e-05, 4.698339762398973e-05, 4.798286681761965e-05, 4.385778811411001e-05, 4.208129394100979e-05, 0.0001138397929025814, 9.082334872800857e-05, 7.164676208049059e-05, 5.857923679286614e-05, 3.957234730478376e-05, 3.493733311188407e-05, 2.9777724193991162e-05, 2.757951369858347e-05, 1.4211145753506571e-05, 1.316496309300419e-05, 1.2700745173788164e-05, 1.2653259545913897e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6579402856441448e-05, "first_to_last_ratio": 1.6134108475690156, "imbalance_abs_log_ratio": 0.47835047692011057, "last_quartile_mean": 1.6474044969072565e-05, "mean": 2.676485377151039e-05, "normalized": [ 1.150774081299089, 1.2446775351438402, 1.1194934771549703, 1.0710868905687188, 0.9358496415246254, 0.8795825855253837, 0.7921399944267447, 0.7509646183368756, 0.7889671267541278, 0.7481474630705149, 0.7042600007590066, 0.6732934483973314, 0.6778913404746827, 0.6508461877769323, 0.6465004342221662, 0.7150440537203975, 1.3113655238548305, 1.3302347281579627, 1.2033980414753724, 1.1364499627034836, 2.9708738613390233, 2.354543738257344, 1.7687393668749452, 1.4507929617425857, 1.0149536514834854, 0.914200457095794, 0.823519211238322, 0.7833869818285496, 0.3624534078136484, 0.3469854230526441, 0.34042254954577716, 0.3381612543808245 ], "peak_layer": 21, "peak_normalized": 2.9708738613390233, "population_cv": 0.5457414223482011, "rest_mean": 2.2244662937333083e-05, "spike_contrast": 2.300501671804746, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.1173884276067835e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 3.0800300010014325e-05, 3.3313612220808864e-05, 2.9963079214212485e-05, 2.866748400265351e-05, 2.5047878807527013e-05, 2.3541899281553924e-05, 2.1201511117396876e-05, 2.0099458197364584e-05, 2.1116589778102934e-05, 2.00240574486088e-05, 1.8849415937438607e-05, 1.8020600691670552e-05, 1.8143662600778043e-05, 1.7419803043594584e-05, 1.7303489585174248e-05, 1.9138049538014457e-05, 3.5098506486974657e-05, 3.560353798093274e-05, 3.220877260901034e-05, 3.0416917070397176e-05, 7.951500447234139e-05, 6.301901885308325e-05, 4.7340050514321774e-05, 3.883026147377677e-05, 2.7165086066816002e-05, 2.4468441552016884e-05, 2.2041371266823262e-05, 2.0967238015145995e-05, 9.70101245911792e-06, 9.28701410884969e-06, 9.111359759117477e-06, 9.050836524693295e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.01899757081992e-05, "first_to_last_ratio": 1.2429080864500646, "imbalance_abs_log_ratio": 0.2174538648629477, "last_quartile_mean": 7.256367280206177e-05, "mean": 0.00010441220069878909, "normalized": [ 0.9923583084473725, 1.071937739818941, 0.9702274874563575, 0.9286563808868895, 0.8280637355418319, 0.7639625829780191, 0.6921006886798987, 0.6629947539328518, 0.7029534904330149, 0.6708874789375917, 0.6281382091404577, 0.5914397520967082, 0.550684307713307, 0.5301298645217033, 0.5304323668326573, 0.5859521330031094, 1.2281373620329439, 1.245755247142365, 1.1474546090559081, 1.1339348253988921, 3.2366950076393746, 2.635045576071926, 2.215617097734964, 1.8966560409115039, 1.197288676699708, 1.059685246423848, 0.9248320715514479, 0.8492170162384304, 0.4187477261886761, 0.38321178810426515, 0.36578292338610474, 0.3610195049989306 ], "peak_layer": 21, "peak_normalized": 3.2366950076393746, "population_cv": 0.6454384585452078, "rest_mean": 8.050837156285222e-05, "spike_contrast": 2.90023103809226, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00023349287803284824, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 0.0001036143148667179, 0.00011192337842658162, 0.00010130358714377508, 9.696305642137304e-05, 8.645995694678277e-05, 7.976701454026625e-05, 7.226375601021573e-05, 6.922474130988121e-05, 7.339692092500627e-05, 7.004883809713647e-05, 6.558529275935143e-05, 6.175352609716356e-05, 5.749816045863554e-05, 5.535202581086196e-05, 5.538361074286513e-05, 6.118055171100423e-05, 0.00012823252473026514, 0.00013007204688619822, 0.00011980826093349606, 0.00011839663056889549, 0.00033795044873841107, 0.00027513090753927827, 0.00023133745708037168, 0.00019803403120022267, 0.0001250115456059575, 0.00011064406862715259, 9.656375186750665e-05, 8.866861753631383e-05, 4.372237162897363e-05, 4.0011986129684374e-05, 3.8192200008779764e-05, 3.7694841012125835e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e3e18e8967d646eb5fe3ca615db63633333413502a9ff39741999ecb78762644", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.051234737038612366, 0.1796528398990631, 0.21105031669139862, 0.23972436785697937, 0.08815877884626389, 0.1068187803030014, 0.12298524379730225, 0.13432784378528595, 0.03348636254668236, 0.051035381853580475, 0.07822930067777634, 0.11166217923164368, 0.038218315690755844, 0.07459890842437744, 0.12532564997673035, 0.36271119117736816, 0.05078785493969917, 0.0794382095336914, 0.10824274271726608, 0.14269457757472992, 0.050489071756601334, 0.06721649318933487, 0.09199884533882141, 0.11444488167762756, 0.024430278688669205, 0.05695623904466629, 0.09002252668142319, 0.130380317568779, 0.030554499477148056, 0.07779927551746368, 0.13734298944473267, 0.22044414281845093 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1110131481473218e-05, "first_to_last_ratio": 1.1615905687058017, "imbalance_abs_log_ratio": 0.1497902458283344, "last_quartile_mean": 9.56458478640343e-06, "mean": 1.3397171542806063e-05, "normalized": [ 0.9679237070104568, 1.0382438480099463, 0.9205891769744879, 0.8794638018815956, 0.7953494837793593, 0.7288626539878529, 0.6666400825149497, 0.6372419269512295, 0.6772452645765212, 0.6436489048118149, 0.6038970887751886, 0.5706326967752795, 0.5512966606689397, 0.5295937419205439, 0.5131637814975486, 0.568852935444849, 1.1361945010844465, 1.1401170159003935, 1.0979049046155025, 1.0452751153521422, 3.411273596228361, 2.8827377384957176, 2.3438854902907558, 1.9385604528250762, 1.2367237086804788, 1.092419222161065, 0.9525920914681678, 0.8037120165341671, 0.4981781184561606, 0.4069912576741014, 0.37037687645015066, 0.35041213820275013 ], "peak_layer": 21, "peak_normalized": 3.411273596228361, "population_cv": 0.6942235633030951, "rest_mean": 1.0016528786191312e-05, "spike_contrast": 3.160041078518313, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.165264242852572e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.2967439943167847e-05, 1.3909530935052317e-05, 1.2333291124377865e-05, 1.1782327419496141e-05, 1.0655433470674325e-05, 9.764698006620165e-06, 8.93109154276317e-06, 8.537239409633912e-06, 9.073170986084733e-06, 8.623074791103136e-06, 8.090512892522383e-06, 7.644864126632456e-06, 7.385815933957929e-06, 7.095058208506089e-06, 6.874943210277706e-06, 7.621020358783426e-06, 1.522179263702128e-05, 1.527434324088972e-05, 1.4708820344822016e-05, 1.4003730029799044e-05, 4.5701417548116297e-05, 3.8620531995547935e-05, 3.140143599011935e-05, 2.5971226932597347e-05, 1.6568599676247686e-05, 1.4635327715950552e-05, 1.2762039659719449e-05, 1.076746775652282e-05, 6.674177711829543e-06, 5.452531695482321e-06, 4.9620025492913555e-06, 4.6945315261837095e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.520103902905248e-05, "first_to_last_ratio": 1.1880781863577057, "imbalance_abs_log_ratio": 0.17233703220725644, "last_quartile_mean": 3.8045508745199186e-05, "mean": 5.402507235885423e-05, "normalized": [ 0.9708715751809724, 1.045181729739689, 0.9250668403006588, 0.8842698161027324, 0.80310215643346, 0.7427420915406581, 0.6768833494137477, 0.6452249292071274, 0.6822190669639766, 0.6476610418037001, 0.6085572495277357, 0.5751764397367072, 0.5529313317984119, 0.5299238822976378, 0.5136471727814265, 0.5689716487649296, 1.141806519457199, 1.159003749171192, 1.1038866764217172, 1.0463951059974672, 3.4229188679498836, 2.87801535441326, 2.3242772711715762, 1.9175102373045263, 1.212626239200128, 1.0731375308479167, 0.9360884367053204, 0.7920928156744456, 0.4948063061071149, 0.406173577795137, 0.36952801888275133, 0.3493029713067937 ], "peak_layer": 21, "peak_normalized": 3.4229188679498836, "population_cv": 0.6923033346437428, "rest_mean": 4.050810336290548e-05, "spike_contrast": 3.135587558841143, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012701670493697748, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 5.2451407100306824e-05, 5.646601857733913e-05, 4.997680298401974e-05, 4.777274079970084e-05, 4.3387652112869546e-05, 4.012669523945078e-05, 3.656867193058133e-05, 3.4858323488151655e-05, 3.685693445731886e-05, 3.498993464745581e-05, 3.287734944024123e-05, 3.1073948775883764e-05, 2.987215520988684e-05, 2.8629176085814834e-05, 2.7749825676437467e-05, 3.073873449466191e-05, 6.168617983348668e-05, 6.261526141315699e-05, 5.9637557569658384e-05, 5.653157131746411e-05, 0.00018492343951947987, 0.00015548498777206987, 0.00012556924775708467, 0.00010359362931922078, 6.551222031703219e-05, 5.7976332755060866e-05, 5.057224552729167e-05, 4.279287168174051e-05, 2.673194649105426e-05, 2.1943556930636987e-05, 1.9963777958764695e-05, 1.8871118300012313e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.040084732696413994, "first_to_last_ratio": 1.188078170049551, "imbalance_abs_log_ratio": 0.1723370184807569, "last_quartile_mean": 0.03373913746327162, "mean": 0.04790997417876497, "normalized": [ 0.9708715136947069, 1.0451817255998777, 0.9250668248545446, 0.8842697810802376, 0.8031021854602314, 0.7427421085449684, 0.6768833571145957, 0.6452249040198467, 0.6822190549510612, 0.6476610399121447, 0.6085572507382631, 0.5751764622807733, 0.5529313119865983, 0.5299238802663323, 0.5136471676571275, 0.5689716073289056, 1.1418065239509545, 1.1590038293019562, 1.1038866872567357, 1.046395186500125, 3.422918937317944, 2.878015431973323, 2.3242771288324753, 1.9175101992153833, 1.2126262668177996, 1.073137513398658, 0.9360884337720428, 0.7920928619984707, 0.4948062938209932, 0.40617357320739117, 0.3695280026437154, 0.349302954501817 ], "peak_layer": 21, "peak_normalized": 3.422918937317944, "population_cv": 0.6923033419361495, "rest_mean": 0.035922990973900865, "spike_contrast": 3.135587556360974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11263968348503113, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.04651442915201187, 0.05007462948560715, 0.04431992769241333, 0.04236534237861633, 0.0384766049683094, 0.03558475524187088, 0.03242946416139603, 0.03091270849108696, 0.03268509730696678, 0.03102942369878292, 0.02915596216917038, 0.027556689456105232, 0.02649092487990856, 0.02538863942027092, 0.024608822539448738, 0.02725941501557827, 0.05470392107963562, 0.05552784353494644, 0.05288718268275261, 0.050132766366004944, 0.16399195790290833, 0.13788564503192902, 0.11135605722665787, 0.09186786413192749, 0.05809689313173294, 0.051413990557193756, 0.04484797269105911, 0.03794914856553078, 0.023706156760454178, 0.019459765404462814, 0.017704077064990997, 0.016735095530748367 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8672166390842904e-06, "first_to_last_ratio": 0.7390827205467719, "imbalance_abs_log_ratio": 0.3023454285110359, "last_quartile_mean": 2.5263973668643303e-06, "mean": 2.8231194875161236e-06, "normalized": [ 1.0710441535522213, 0.7806782718477623, 0.7446651107672629, 0.7227944286323706, 0.5070907376718641, 0.49705683909512793, 0.484348171178412, 0.483537617930567, 0.5122490344314135, 0.5011923600503864, 0.49494085537056437, 0.4929156803489962, 0.48753662380682133, 0.48921410821124456, 0.49216500849421285, 0.5089367920233998, 0.820710291882377, 0.8409358666404328, 0.8609799850722372, 0.8853668132738947, 3.599560016732762, 3.2818994953739233, 2.821405059000161, 2.4596114274457115, 1.4921895479857374, 1.353669348387604, 1.2002458902354434, 1.0429447595206092, 0.6391234507721919, 0.5173406844488531, 0.47262998257826977, 0.441021587237165 ], "peak_layer": 21, "peak_normalized": 3.599560016732762, "population_cv": 0.8312440892614361, "rest_mean": 1.918187822243285e-06, "spike_contrast": 4.0192886174061115, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.709750479989453e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.0236856218834873e-06, 2.2039480427338276e-06, 2.1022785858804127e-06, 2.0405350369401276e-06, 1.431577743460366e-06, 1.4032508488526219e-06, 1.3673727607965702e-06, 1.3650844721269095e-06, 1.446140231564641e-06, 1.4149259186524432e-06, 1.3972771739645395e-06, 1.3915598628955195e-06, 1.3763741435468546e-06, 1.3811098824589862e-06, 1.3894406265535508e-06, 1.4367893754752004e-06, 2.3169632186181843e-06, 2.374062432863866e-06, 2.430649374218774e-06, 2.499496304153581e-06, 1.0161988029722124e-05, 9.265194421459455e-06, 7.965163604239933e-06, 6.943776952539338e-06, 4.212629391986411e-06, 3.821570317086298e-06, 3.3884375625348184e-06, 2.944357675005449e-06, 1.8043218688035267e-06, 1.4605145679524867e-06, 1.3342909142011194e-06, 1.2450566373445326e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.515827052069653e-05, "first_to_last_ratio": 1.3729523213289132, "imbalance_abs_log_ratio": 0.31696340027531067, "last_quartile_mean": 2.5607786938053323e-05, "mean": 3.930294792553468e-05, "normalized": [ 1.0457356769392694, 1.1170113876105092, 0.9973655731270212, 0.9523397521545555, 0.848322559088178, 0.7912242750828141, 0.7196584735613146, 0.6847053492610473, 0.7251119779087116, 0.6895313261932131, 0.6494379753000808, 0.613786235590645, 0.6090154259037742, 0.5840032818494388, 0.5660492129431545, 0.6192857434966186, 1.2024027857653905, 1.2099521826215391, 1.1323058584311179, 1.0603698069411953, 3.3091113142819246, 2.746844343609951, 2.155778589894086, 1.758260822046451, 1.144969795788572, 0.9973957484994478, 0.8715780521673296, 0.7496724170015708, 0.433790833285331, 0.35920735567744455, 0.33316471339849535, 0.32261115457980727 ], "peak_layer": 21, "peak_normalized": 3.3091113142819246, "population_cv": 0.6495478539134593, "rest_mean": 3.040160919664669e-05, "spike_contrast": 2.873866856730952, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.737017706152982e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 4.1100494854617864e-05, 4.390184039948508e-05, 3.919940718333237e-05, 3.74297596863471e-05, 3.334157736389898e-05, 3.109744648099877e-05, 2.8284699510550126e-05, 2.691093868634198e-05, 2.849903830792755e-05, 2.7100613806396723e-05, 2.5524826924083754e-05, 2.4123608454829082e-05, 2.3936101570143364e-05, 2.295305057486985e-05, 2.224740273959469e-05, 2.433975532767363e-05, 4.725797407445498e-05, 4.755468762596138e-05, 4.4502958189696074e-05, 4.1675659304019064e-05, 0.0001300578296650201, 0.0001079590801964514, 8.472845365758985e-05, 6.910483352839947e-05, 4.5000688260188326e-05, 3.920059316442348e-05, 3.425558679737151e-05, 2.9464335966622457e-05, 1.704925853118766e-05, 1.4117907994659618e-05, 1.309435538132675e-05, 1.2679569408646785e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6777070615935372e-05, "first_to_last_ratio": 1.4895983067726413, "imbalance_abs_log_ratio": 0.39850649084066486, "last_quartile_mean": 1.7976034541788977e-05, "mean": 2.8343781877993024e-05, "normalized": [ 1.1067878423232929, 1.1808830950582898, 1.0583921093539743, 1.0130981930448324, 0.8884743193198895, 0.8335235896508596, 0.7557776874483318, 0.7208607029428713, 0.7654002308457403, 0.723786998353269, 0.6828269487786236, 0.6470347967107349, 0.6602591516703425, 0.6315002997298088, 0.6140621512563034, 0.6701035513604336, 1.2453501569114247, 1.2538344754875952, 1.1579706088704604, 1.0596025963058344, 3.1377161786254733, 2.5783790501879347, 1.9479449698525808, 1.5927151172754295, 1.103279856014175, 0.9422718711690712, 0.854272511709325, 0.7657265014248548, 0.4191617190143686, 0.34148416420892125, 0.32691118354419935, 0.32060737155075375 ], "peak_layer": 21, "peak_normalized": 3.1377161786254733, "population_cv": 0.5950537042380278, "rest_mean": 2.2716979367908573e-05, "spike_contrast": 2.5852255478742316, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.872851543244906e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 3.137055318802595e-05, 3.347069286974147e-05, 2.9998835088917986e-05, 2.87150342046516e-05, 2.518272231100127e-05, 2.3625210815225728e-05, 2.1421597921289504e-05, 2.0431918528629467e-05, 2.169433719245717e-05, 2.051486080745235e-05, 1.9353898096596822e-05, 1.833941314544063e-05, 1.8714241377892904e-05, 1.7899106751428917e-05, 1.7404843674739823e-05, 1.8993268895428628e-05, 3.529793320922181e-05, 3.5538410884328187e-05, 3.282126635895111e-05, 3.0033144867047668e-05, 8.89347429620102e-05, 7.308101339731365e-05, 5.521212733583525e-05, 4.514356987783685e-05, 3.127112358924933e-05, 2.67075483861845e-05, 2.421331373625435e-05, 2.1703584934584796e-05, 1.1880628335347865e-05, 9.678952665126417e-06, 9.265899279853329e-06, 9.08722540771123e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.124297230300726e-05, "first_to_last_ratio": 1.129611466147561, "imbalance_abs_log_ratio": 0.12187373833170514, "last_quartile_mean": 8.077376605797326e-05, "mean": 0.0001132043312281894, "normalized": [ 0.9389217132455487, 0.9994043961548372, 0.8982931506377284, 0.859853269169116, 0.7637743898798451, 0.7087638918644622, 0.6499465810809194, 0.6290624894037778, 0.6694605698054724, 0.6398149406563152, 0.5998145159738899, 0.5645121104930153, 0.5238242883776524, 0.5044864713561084, 0.48955416809270236, 0.5348544623331092, 1.1363983661263592, 1.1357195169737937, 1.078307851992808, 1.031529476521427, 3.518987411841691, 2.9635435485495347, 2.411411019127874, 2.041586444021295, 1.2317185051916162, 1.0997550375371743, 0.974217691447948, 0.8518825440407063, 0.4721291734182047, 0.38687023180546226, 0.35404975204423306, 0.3375520208353729 ], "peak_layer": 21, "peak_normalized": 3.518987411841691, "population_cv": 0.7265349466170719, "rest_mean": 8.315383510543379e-05, "spike_contrast": 3.312859953383778, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00027547701029106975, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00010629000462358817, 0.00011313690629322082, 0.00010169067536480725, 9.733911429066211e-05, 8.646256901556626e-05, 8.023514237720519e-05, 7.357676804531366e-05, 7.121259841369465e-05, 7.578583608847111e-05, 7.242982246680185e-05, 6.790160114178434e-05, 6.390521593857557e-05, 5.929917824687436e-05, 5.711005360353738e-05, 5.541965219890699e-05, 6.054784171283245e-05, 0.00012864521704614162, 0.00012856836838182062, 0.00012206911924295127, 0.00011677360453177243, 0.00039836461655795574, 0.0003354859654791653, 0.0002729821717366576, 0.00023111642803996801, 0.00013943586964160204, 0.00012449703353922814, 0.00011028566223103553, 9.643679368309677e-05, 5.344706733012572e-05, 4.3795385863631964e-05, 4.007996540167369e-05, 3.8212350773392245e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "1266324f6216c67f6f92c122a2698de3ea217c2cc1431a3715e69a04aa62551d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0955304205417633, 0.19925470650196075, 0.2359117865562439, 0.25103330612182617, 0.09304249286651611, 0.11085781455039978, 0.12910984456539154, 0.14378803968429565, 0.04010729491710663, 0.06472750008106232, 0.09987020492553711, 0.14899125695228577, 0.05702618882060051, 0.10170809924602509, 0.15923865139484406, 0.38537684082984924, 0.05919196084141731, 0.08540526777505875, 0.1145314872264862, 0.15640178322792053, 0.06262174993753433, 0.08427268266677856, 0.11030438542366028, 0.13170360028743744, 0.05311359465122223, 0.08614693582057953, 0.12623678147792816, 0.17503507435321808, 0.07505694776773453, 0.13522638380527496, 0.2187158316373825, 0.3292153775691986 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0990825558110373e-05, "first_to_last_ratio": 1.2944181557270227, "imbalance_abs_log_ratio": 0.25806129358885044, "last_quartile_mean": 8.490938967042894e-06, "mean": 1.2353476122939355e-05, "normalized": [ 1.0282384798178228, 1.117858133996928, 0.9935843345501945, 0.9494663755935344, 0.8590058043818248, 0.7847999930233409, 0.709132552716486, 0.6754740791306071, 0.7068824996302968, 0.674955776214346, 0.6307861342112064, 0.6026941529611411, 0.5785889076796064, 0.556995885467898, 0.5565612914175161, 0.6240017404560392, 1.2244356436045847, 1.2505073846311183, 1.1714347530624865, 1.1440952313231665, 3.1783392045284105, 2.5998686819467225, 2.1163742238669494, 1.767263048891068, 1.1859101289408378, 1.03626731593003, 0.8733325362846178, 0.8036687223427385, 0.4425832407384721, 0.40163477037506473, 0.38000515774441007, 0.3752538145405344 ], "peak_layer": 21, "peak_normalized": 3.1783392045284105, "population_cv": 0.6201793460523679, "rest_mean": 9.677916666448624e-06, "spike_contrast": 2.7693457292213166, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.68014971879893e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 1.2702319509116933e-05, 1.3809433767164592e-05, 1.2274220352992415e-05, 1.1729210200428497e-05, 1.0611707693897188e-05, 9.695007975096814e-06, 8.760252057982143e-06, 8.344452908204403e-06, 8.732456080906559e-06, 8.338050065503921e-06, 7.792401447659358e-06, 7.445367828040617e-06, 7.147584256017581e-06, 6.880835371703142e-06, 6.875466624478577e-06, 7.708590601396281e-06, 1.5126036487345118e-05, 1.544811311759986e-05, 1.4471291251538787e-05, 1.4133553122519515e-05, 3.926353747374378e-05, 3.211741568520665e-05, 2.614457844174467e-05, 2.1831841877428815e-05, 1.4650112461822573e-05, 1.2801503544324078e-05, 1.0788692634378094e-05, 9.928102372214198e-06, 5.467441496875836e-06, 4.961585545970593e-06, 4.694384642789373e-06, 4.635689037968405e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.4711176087730564e-05, "first_to_last_ratio": 1.3207760209544162, "imbalance_abs_log_ratio": 0.27821945851834845, "last_quartile_mean": 3.385220156815194e-05, "mean": 4.981010016535947e-05, "normalized": [ 1.0309026668033479, 1.1248865671658381, 0.9983068512481783, 0.9550523854303026, 0.8675947035501784, 0.799947715571839, 0.7205411910772573, 0.68382974224492, 0.7120277827066004, 0.6791934284063869, 0.6349499689734955, 0.6062730250974693, 0.5788392434931497, 0.5568995584252106, 0.5581244614543389, 0.6247854138149606, 1.2301897371875206, 1.271033179578376, 1.1767202476646232, 1.146019522378179, 3.189751492495938, 2.584287570955602, 2.0948183363362958, 1.7380232415485222, 1.165593138274326, 1.019662423146035, 0.8607718088943986, 0.7949347500773989, 0.4416990390008851, 0.4009654076948961, 0.3788881940118565, 0.3744872052916737 ], "peak_layer": 21, "peak_normalized": 3.189751492495938, "population_cv": 0.6175097196118516, "rest_mean": 3.9160933603718013e-05, "spike_contrast": 2.740373880872814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010731559959822334, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 5.134936509421095e-05, 5.603071258519776e-05, 4.972576425643638e-05, 4.7571254981448874e-05, 4.3214979086769745e-05, 3.9845475839683786e-05, 3.5890228900825605e-05, 3.406162795727141e-05, 3.5466175177134573e-05, 3.383069270057604e-05, 3.1626921554561704e-05, 3.0198520107660443e-05, 2.883204069803469e-05, 2.77392227872042e-05, 2.7800235329777934e-05, 3.112062404397875e-05, 6.127587403170764e-05, 6.331028998829424e-05, 5.8612553402781487e-05, 5.708334720111452e-05, 0.00015888184134382755, 0.00012872362276539207, 0.00010434311116114259, 8.657111175125465e-05, 5.805831096949987e-05, 5.078948743175715e-05, 4.287513002054766e-05, 3.959577952628024e-05, 2.2001073375577107e-05, 1.997212712012697e-05, 1.8872458895202726e-05, 1.86532452062238e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03965031821280718, "first_to_last_ratio": 1.3207760013357848, "imbalance_abs_log_ratio": 0.2782194436644813, "last_quartile_mean": 0.0300204714294523, "mean": 0.04417209519306198, "normalized": [ 1.0309027129185282, 1.124886561555575, 0.9983068303969317, 0.9550523327081637, 0.8675947279179665, 0.7999476938642114, 0.7205412090579838, 0.6838297380461792, 0.7120277600555036, 0.6791934601432961, 0.6349499561794323, 0.6062730331610577, 0.5788392624406322, 0.5568995493471454, 0.5581244851058754, 0.6247854224706602, 1.2301896279139695, 1.2710332172386067, 1.1767202901940028, 1.1460195181019854, 3.189751438287289, 2.5842874809086247, 2.0948183490366055, 1.7380233083860965, 1.1655931894927514, 1.019662433533796, 0.8607717906221712, 0.7949347721458104, 0.4416990408526936, 0.4009654254500078, 0.3788882027861667, 0.37448717968028017 ], "peak_layer": 21, "peak_normalized": 3.189751438287289, "population_cv": 0.6175097084858858, "rest_mean": 0.034728307759872186, "spike_contrast": 2.7403738756960694, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09516854733228683, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 0.045537132769823074, 0.04968859627842903, 0.044097304344177246, 0.042186662554740906, 0.038323476910591125, 0.03533536568284035, 0.031827814877033234, 0.030206192284822464, 0.0314517579972744, 0.03000139817595482, 0.028047069907188416, 0.026780350133776665, 0.025568543002009392, 0.02459941990673542, 0.02465352788567543, 0.027598081156611443, 0.05434005334973335, 0.05614420026540756, 0.05197820067405701, 0.050622083246707916, 0.14089800417423248, 0.11415339261293411, 0.09253251552581787, 0.07677213102579117, 0.05148669332265854, 0.04504062607884407, 0.03802209347486496, 0.03511393442749977, 0.019510772079229355, 0.01771148294210434, 0.016736285760998726, 0.01654188334941864 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8471929479346727e-06, "first_to_last_ratio": 0.824461579019105, "imbalance_abs_log_ratio": 0.19302473719608554, "last_quartile_mean": 2.2404839654655007e-06, "mean": 2.553080481249026e-06, "normalized": [ 1.1469614368871983, 0.8593837713333818, 0.8257379810924192, 0.8007786067207996, 0.5570026135445557, 0.5466338814455962, 0.5274997830623889, 0.5241248198100039, 0.5489562615578096, 0.5407197695419355, 0.5308505223860043, 0.5295945294910184, 0.5283032694056181, 0.5289080215685197, 0.5448229646529533, 0.566591415289843, 0.8991703275174108, 0.9131167201550765, 0.9272471968396779, 0.9661518060876617, 3.3508739801606664, 2.970736593720056, 2.5767839088445323, 2.268561489055477, 1.4764458758023058, 1.3194996647146926, 1.1214564007083334, 1.0614159683161588, 0.5682809452629535, 0.5158177934133008, 0.48288389278490795, 0.4746877888267432 ], "peak_layer": 21, "peak_normalized": 3.3508739801606664, "population_cv": 0.7395096569005253, "rest_mean": 1.8303315898635816e-06, "spike_contrast": 3.5271884780241365, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.455924494730425e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.9282848572620424e-06, 2.1940759324934334e-06, 2.108175522153033e-06, 2.0444522306206636e-06, 1.4220725006452994e-06, 1.3956002931081457e-06, 1.3467493999996805e-06, 1.3381328471950837e-06, 1.4015295164426789e-06, 1.380501089442987e-06, 1.3553041071645566e-06, 1.3520974562197807e-06, 1.3488007652995293e-06, 1.3503447462426266e-06, 1.390976876791683e-06, 1.446553483219759e-06, 2.2956542125029955e-06, 2.3312604753300548e-06, 2.367336719544255e-06, 2.466663318045903e-06, 8.555050953873433e-06, 7.584529612358892e-06, 6.578736702067545e-06, 5.791820058220765e-06, 3.7694851471314905e-06, 3.368788838997716e-06, 2.8631684472202323e-06, 2.7098803911940195e-06, 1.4508669892165926e-06, 1.3169243402444408e-06, 1.2328414413786959e-06, 1.2119161283408175e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.4829904052458005e-05, "first_to_last_ratio": 1.509444580974737, "imbalance_abs_log_ratio": 0.4117417559927783, "last_quartile_mean": 2.3074649107002188e-05, "mean": 3.6670617419076734e-05, "normalized": [ 1.0984878069834736, 1.188723837036249, 1.0648721392453866, 1.0174200831414708, 0.9061105203098224, 0.8430649229703882, 0.7598681195362218, 0.7198856496460319, 0.7526438720220634, 0.718650325146632, 0.6733539856914239, 0.6414557384537676, 0.6322823726903553, 0.6081123440925361, 0.6055674387182303, 0.669931198330675, 1.2812282331891511, 1.3084793805341686, 1.1959962991108561, 1.1475497928959553, 3.1043863400982548, 2.476737525556709, 1.953803593412093, 1.5974619795855607, 1.0791244029968152, 0.9527382661611272, 0.8120339425793232, 0.7520941665087985, 0.38753474791935355, 0.35900363296340954, 0.3463461699660851, 0.3450511725076112 ], "peak_layer": 21, "peak_normalized": 3.1043863400982548, "population_cv": 0.5864654402354135, "rest_mean": 2.9592490372360096e-05, "spike_contrast": 2.5307942159980388, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.489250347134658e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 4.028222610941157e-05, 4.3591237044893205e-05, 3.904951881850138e-05, 3.730942262336612e-05, 3.322763222968206e-05, 3.09157112496905e-05, 2.7864833100466058e-05, 2.639865124365315e-05, 2.759991548373364e-05, 2.6353351131547242e-05, 2.4692306396900676e-05, 2.352257797610946e-05, 2.318618498975411e-05, 2.229985511803534e-05, 2.220653186668642e-05, 2.45667906710878e-05, 4.6983430365798995e-05, 4.798274676431902e-05, 4.385792271932587e-05, 4.208135942462832e-05, 0.00011383976379875094, 9.082349424716085e-05, 7.164718408603221e-05, 5.857991709490307e-05, 3.957215812988579e-05, 3.49375004589092e-05, 2.9777786039630882e-05, 2.7579757443163544e-05, 1.4211138477548957e-05, 1.3164884876459837e-05, 1.270072789338883e-05, 1.2653239537030458e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6578757797324215e-05, "first_to_last_ratio": 1.6134215502902036, "imbalance_abs_log_ratio": 0.47835711049748203, "last_quartile_mean": 1.6473535879413248e-05, "mean": 2.6764625772557338e-05, "normalized": [ 1.1507208152122448, 1.2447207601116925, 1.1194216627013838, 1.0711038984456216, 0.9358933619627398, 0.8793303259328489, 0.7922968034824025, 0.7509560638170416, 0.7890091881742389, 0.7482141869411878, 0.7043635941867609, 0.6733903895614985, 0.6779168243147607, 0.6506778841853507, 0.6465418257154243, 0.7151576615308665, 1.3113688112983555, 1.3303637709400895, 1.2034267786084025, 1.1362866793208775, 2.97087008129982, 2.355025940458153, 1.7683296688974177, 1.4506403077495817, 1.0150838143878007, 0.9142558185771078, 0.8233714760378564, 0.7833275277830687, 0.36254973990814826, 0.34689941606582414, 0.3403766524569371, 0.33810826993449583 ], "peak_layer": 21, "peak_normalized": 2.97087008129982, "population_cv": 0.5457582329285832, "rest_mean": 2.2244427614000245e-05, "spike_contrast": 2.300517537100295, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.117369582876563e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 3.0798611987847835e-05, 3.3314485335722566e-05, 2.996090188389644e-05, 2.866769500542432e-05, 2.5048835595953278e-05, 2.3534947104053572e-05, 2.1205527445999905e-05, 2.0099058019695804e-05, 2.1117535652592778e-05, 2.002567271119915e-05, 1.88520280062221e-05, 1.802304177545011e-05, 1.814419010770507e-05, 1.7415150068700314e-05, 1.730445001157932e-05, 1.9140927179250866e-05, 3.5098295484203845e-05, 3.5606688470579684e-05, 3.22092673741281e-05, 3.041228774236515e-05, 7.951422594487667e-05, 6.303138798102736e-05, 4.732868183054961e-05, 3.882584496750496e-05, 2.7168338419869542e-05, 2.4469714844599366e-05, 2.2037229427951388e-05, 2.0965468138456345e-05, 9.703508112579584e-06, 9.284633051720448e-06, 9.11005372472573e-06, 9.04934131540358e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.019189837999875e-05, "first_to_last_ratio": 1.2430263788421747, "imbalance_abs_log_ratio": 0.21754903421986438, "last_quartile_mean": 7.255831405927893e-05, "mean": 0.00010441271081162995, "normalized": [ 0.9923145762328835, 1.0717706255029942, 0.9701026808081262, 0.9288730924893803, 0.8283807269361432, 0.7640642833999716, 0.6920059508859676, 0.6629032941440701, 0.7029452478860042, 0.6708830166494891, 0.6281695645345438, 0.5913612548049529, 0.5507136373932213, 0.5301664721346755, 0.5303584531978167, 0.5861270357139448, 1.228189060760272, 1.2457938984618147, 1.1471984869923635, 1.1338859416919997, 3.2374008483056524, 2.634554108633911, 2.2155512224100757, 1.8969393106516845, 1.1972085435140079, 1.059610663427442, 0.9247236535108656, 0.8491302911049953, 0.4186534528150795, 0.3832452808390877, 0.36580012539024503, 0.36097519877631845 ], "peak_layer": 21, "peak_normalized": 3.2374008483056524, "population_cv": 0.6454817160990234, "rest_mean": 8.050740507314913e-05, "spike_contrast": 2.9003712340158865, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00023350136179942638, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 0.00010361025488236919, 0.00011190647637704387, 0.00010129105066880584, 9.698615758679807e-05, 8.649347728351131e-05, 7.97780230641365e-05, 7.225421722978354e-05, 6.921552994754165e-05, 7.339641888393089e-05, 7.004871440585703e-05, 6.558888708241284e-05, 6.174563168315217e-05, 5.7501503761159256e-05, 5.535611853701994e-05, 5.5376163800247014e-05, 6.119911267887801e-05, 0.00012823854922316968, 0.00013007671805098653, 0.00011978210386587307, 0.00011839210492325947, 0.00033802579855546355, 0.00027508093626238406, 0.00023133170907385647, 0.000198064575670287, 0.0001250037894351408, 0.00011063682177336887, 9.65529034147039e-05, 8.865999552654102e-05, 4.371274189907126e-05, 4.001567867817357e-05, 3.819418270722963e-05, 3.7690399040002376e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "9ec6f7109f4e916c83054b81e32dd86c0125957a9aa2bade5062f6ae3f8e89ac", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0558863990008831, 0.061753273010253906, 0.11003358662128448, 0.1138310506939888, 0.13082429766654968, 0.10403942316770554, 0.1062953919172287, 0.10766621679067612, 0.11238158494234085, 0.08696430176496506, 0.08355452120304108, 0.08609789609909058, 0.100697360932827, 0.07805952429771423, 0.08197034895420074, 0.10499614477157593, 0.13756641745567322, 0.09882861375808716, 0.10110001266002655, 0.10269922018051147, 0.10988812148571014, 0.0575832724571228, 0.06622003018856049, 0.06460635364055634, 0.07165247201919556, 0.055589381605386734, 0.05917651578783989, 0.06219335272908211, 0.06750614196062088, 0.06024058535695076, 0.06803743541240692, 0.06952574104070663 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.596624018584407e-06, "first_to_last_ratio": 1.8980506308307645, "imbalance_abs_log_ratio": 0.640827375717486, "last_quartile_mean": 3.4754731572661512e-06, "mean": 8.708262916101717e-06, "normalized": [ 1.0256731976862243, 0.9753411624231779, 0.9035749226639593, 0.5582866185427084, 0.48593023814787983, 0.8667266687937877, 0.7602289215288303, 0.4843437880540975, 0.4388532976888435, 0.471538921660211, 0.6133103789102933, 0.5130639656537879, 0.3167954672221293, 0.31280913276495154, 0.23475435209498255, 1.0312032138821217, 1.0496881251460148, 1.5459688796414424, 2.660629290297128, 2.4820591361597244, 4.429437174760716, 2.8054906698151227, 2.505482199716851, 1.336005290721962, 0.7761527446294757, 0.5408170248947934, 0.6194156007606676, 0.405832393835082, 0.32631761431893097, 0.23366629172124823, 0.18561467402758638, 0.10498864183526842 ], "peak_layer": 21, "peak_normalized": 4.429437174760716, "population_cv": 0.9558477179995892, "rest_mean": 6.498116083211816e-06, "spike_contrast": 3.176775476055201, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0643055813707177e-05, "top_five_layers": [ 21, 22, 19, 23, 20 ], "top_five_spike_overlap": 3 }, "values": [ 8.931831871450413e-06, 8.493527275277302e-06, 7.868567990954034e-06, 4.861706656811293e-06, 4.231608272675658e-06, 7.547683708253317e-06, 6.620273325097514e-06, 4.217793048155727e-06, 3.821649897872703e-06, 4.106284904992208e-06, 5.340868028724799e-06, 4.467895905690966e-06, 2.7587382191995857e-06, 2.7240241706749657e-06, 2.044302618742222e-06, 8.979988706414588e-06, 9.140960173681378e-06, 1.3462703464028891e-05, 2.3169459382188506e-05, 2.161442353099119e-05, 3.85727034881711e-05, 2.44309503614204e-05, 2.1818397726747207e-05, 1.1634285328909755e-05, 6.758942163287429e-06, 4.709576842287788e-06, 5.394033905758988e-06, 3.534095185386832e-06, 2.841659579644329e-06, 2.0348275029391516e-06, 1.616381382518739e-06, 9.142686963059532e-07 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.6033237190858927e-05, "first_to_last_ratio": 1.8862787219924644, "imbalance_abs_log_ratio": 0.6346059580177699, "last_quartile_mean": 1.3801373512478676e-05, "mean": 3.5272740866787444e-05, "normalized": [ 0.9568267063421432, 0.9174711042795557, 0.911235657888503, 0.5484831467374834, 0.4781383342683516, 0.8432205386406871, 0.7712046240634284, 0.47786316065782963, 0.4398762585196919, 0.51209123046888, 0.6108901597314282, 0.4846724182865656, 0.3294043935332117, 0.3091278217388424, 0.22447937051100367, 0.999498418571711, 1.0065120452069123, 1.5624889800277921, 2.795100908402575, 2.4945789402126657, 4.378703171587316, 2.953659490750943, 2.5415441033638264, 1.3227217970755467, 0.7766017604446148, 0.5134703473853324, 0.605157823112537, 0.39031143703269466, 0.3253712384885135, 0.23092588991327564, 0.1825610852772617, 0.10580763747887693 ], "peak_layer": 21, "peak_normalized": 4.378703171587316, "population_cv": 0.9707222592180655, "rest_mean": 2.616292804473401e-05, "spike_contrast": 3.2284509578382985, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.4465730105876e-05, "top_five_layers": [ 21, 22, 19, 23, 20 ], "top_five_spike_overlap": 3 }, "values": [ 3.3749900467228144e-05, 3.236172051401809e-05, 3.2141779229277745e-05, 1.9346503904671408e-05, 1.6865249563124962e-05, 2.9742699553025886e-05, 2.7202500859857537e-05, 1.685554343566764e-05, 1.5515641280217096e-05, 1.8062861272483133e-05, 2.154777030227706e-05, 1.709572461550124e-05, 1.161899581347825e-05, 1.0903785550908651e-05, 7.9180026659742e-06, 3.525504871504381e-05, 3.5502438549883664e-05, 5.511326889973134e-05, 9.859087003860623e-05, 8.79906365298666e-05, 0.0001544488623039797, 0.00010418366582598537, 8.964722655946389e-05, 4.665602318709716e-05, 2.739287265285384e-05, 1.811150650610216e-05, 2.1345575078157708e-05, 1.3767354175797664e-05, 1.1476735380711034e-05, 8.145389074343257e-06, 6.439429853344336e-06, 3.732125378519413e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.023086535395123065, "first_to_last_ratio": 1.8862787345100174, "imbalance_abs_log_ratio": 0.6346059646538798, "last_quartile_mean": 0.012239196134032682, "mean": 0.03128021959128091, "normalized": [ 0.9568267121752291, 0.9174711066988445, 0.9112356957360986, 0.5484831570712991, 0.4781383082120972, 0.8432205065508228, 0.7712046480612316, 0.4778631710159881, 0.43987625474046216, 0.512091268376117, 0.6108901444890714, 0.4846724370071731, 0.3294043980010242, 0.30912781978929554, 0.22447935000902958, 0.9994983743937245, 1.0065119463517638, 1.5624889224293914, 2.7951009409274206, 2.4945790708322866, 4.378703107551428, 2.9536595207599716, 2.541544076160719, 1.322721846992968, 0.7766017554969081, 0.5134703187656182, 0.6051578466461017, 0.390311454495132, 0.3253712457748438, 0.23092588507713302, 0.18256107720838338, 0.10580763220242217 ], "peak_layer": 21, "peak_normalized": 4.378703107551428, "population_cv": 0.9707222603656619, "rest_mean": 0.02320154643341623, "spike_contrast": 3.2284509508326362, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07490505464375019, "top_five_layers": [ 21, 22, 19, 23, 20 ], "top_five_spike_overlap": 3 }, "values": [ 0.0299297496676445, 0.028698697686195374, 0.028503652662038803, 0.017156673595309258, 0.014956271275877953, 0.02637612260878086, 0.024123450741171837, 0.014947664923965931, 0.013759425841271877, 0.016018327325582504, 0.01910877786576748, 0.01516066025942564, 0.010303841903805733, 0.009669586084783077, 0.007021763361990452, 0.031264528632164, 0.03148391470313072, 0.04887499660253525, 0.08743137121200562, 0.07803098112344742, 0.1369667947292328, 0.09239111840724945, 0.07950005680322647, 0.04137502983212471, 0.024292273446917534, 0.016061464324593544, 0.01892947033047676, 0.012209028005599976, 0.010177684016525745, 0.007223412394523621, 0.005710550583899021, 0.003309685969725251 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8957251032579734e-06, "first_to_last_ratio": 1.3390904119295601, "imbalance_abs_log_ratio": 0.2919905864082896, "last_quartile_mean": 2.1624567523303995e-06, "mean": 2.6915775031000067e-06, "normalized": [ 4.8336725838512145, 1.7374668605804229, 0.3213508492275624, 0.4363891970704281, 0.32589544632371475, 0.307241753778847, 0.3145233096085374, 0.3302345766920988, 0.3490931446328018, 0.3693094318109904, 0.3998151480537758, 0.5074504002356933, 0.29839576596304346, 0.36719305482228454, 0.37294658781774515, 0.5123164707134974, 0.3979121151789476, 0.6707129654139522, 0.9550170379823126, 1.0953232949941911, 2.212763417999681, 3.415237241942596, 2.685461662904724, 2.356948728016963, 1.0448937474521496, 1.0864393774586762, 1.3084278349629268, 0.9447949362821033, 0.7371038712092777, 0.5351342382555423, 0.3756737262173936, 0.39486122254590533 ], "peak_layer": 1, "peak_normalized": 4.8336725838512145, "population_cv": 1.0373807348560817, "rest_mean": 2.022141824521963e-06, "spike_contrast": 3.1187378110396953, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.3065301674214425e-06, "top_five_layers": [ 1, 22, 23, 24, 21 ], "top_five_spike_overlap": 4 }, "values": [ 1.301020438404521e-05, 4.676526714320062e-06, 8.649407163829892e-07, 1.1745753454306396e-06, 8.771728516876465e-07, 8.26964992484136e-07, 8.465638643428974e-07, 8.888519573702069e-07, 9.396112545800861e-07, 9.940249583451077e-07, 1.0761334579001414e-06, 1.3658420812134864e-06, 8.031553306864225e-07, 9.883285656542284e-07, 1.0038146456281538e-06, 1.3789394870400429e-06, 1.071011297426594e-06, 1.8052759287456865e-06, 2.570502374510397e-06, 2.9481475394277368e-06, 5.955824235570617e-06, 9.192375728162006e-06, 7.228128197311889e-06, 6.343910172290634e-06, 2.812412503772066e-06, 2.9242357868497493e-06, 3.5217349250160623e-06, 2.5429887955397135e-06, 1.9839721971948165e-06, 1.4403552768271766e-06, 1.0111549499924877e-06, 1.062799583451124e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.7218609059455048e-05, "first_to_last_ratio": 2.059812082463918, "imbalance_abs_log_ratio": 0.7226147565356063, "last_quartile_mean": 8.35931064102624e-06, "mean": 2.3338234882430697e-05, "normalized": [ 0.7455840782742903, 0.932126430786308, 0.9325399041631259, 0.5978598612170376, 0.5053742064366669, 0.8701260784473666, 0.8098658216052845, 0.5088068538375754, 0.46804816039298675, 0.5744347433377897, 0.6330397816872262, 0.500448729191671, 0.3576625767404614, 0.35542276690703867, 0.26113068942082235, 1.0318778766826857, 1.0488894418321313, 1.6253919807257495, 2.886863167031188, 2.3788037608248445, 3.8936434825321258, 3.128031704756627, 2.6914081834944965, 1.3971722912336362, 0.7333705189828876, 0.5031325260356582, 0.6161181986326979, 0.3294770981758386, 0.28015866797271904, 0.16234546962292012, 0.14387918744395148, 0.09696576157419173 ], "peak_layer": 21, "peak_normalized": 3.8936434825321258, "population_cv": 0.9430038169349005, "rest_mean": 1.7422747650589036e-05, "spike_contrast": 3.172970592414376, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.528186593437567e-05, "top_five_layers": [ 21, 22, 19, 23, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.7400616343365982e-05, 2.1754185581812635e-05, 2.1763835320598446e-05, 1.3952993867860641e-05, 1.1794541933340952e-05, 2.030720679613296e-05, 1.890083876787685e-05, 1.1874653864651918e-05, 1.0923417903541122e-05, 1.3406292964646127e-05, 1.4774031114939135e-05, 1.167958998848917e-05, 8.347213224624284e-06, 8.294940016639885e-06, 6.094329364714213e-06, 2.4082208256004378e-05, 2.447922815917991e-05, 3.793377982219681e-05, 6.737429066561162e-05, 5.551708090933971e-05, 9.08707661437802e-05, 7.300273864530027e-05, 6.28127163508907e-05, 3.260753510403447e-05, 1.7115573427872732e-05, 1.1742225069610868e-05, 1.4379111235029995e-05, 7.689413905609399e-06, 6.538408797496231e-06, 3.7888567021582276e-06, 3.357886271260213e-06, 2.26300971917226e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.1091838814536459e-05, "first_to_last_ratio": 2.147117923315226, "imbalance_abs_log_ratio": 0.764126442081038, "last_quartile_mean": 5.165919716887402e-06, "mean": 1.5115809230081823e-05, "normalized": [ 0.5373176711890457, 0.9499619969417363, 0.9455847427341147, 0.6485396392348263, 0.5327547371672011, 0.8976617814793185, 0.8429317236262398, 0.5155725553175506, 0.4928615554351285, 0.6510671951764587, 0.6749615278217853, 0.4894485606904807, 0.38754415734688374, 0.4151562078149955, 0.30831642778913926, 1.0584316197438957, 1.0910562730904934, 1.7032600694319715, 2.987128042597488, 2.249255714777125, 3.2136369586789724, 3.297496124030594, 2.9011704426376874, 1.4748356308081663, 0.7355649883790728, 0.5038469292009693, 0.6237518052151605, 0.2779007685642119, 0.24681129799141374, 0.13171952134177142, 0.12082312918165286, 0.09363020456444852 ], "peak_layer": 22, "peak_normalized": 3.297496124030594, "population_cv": 0.9134577450694712, "rest_mean": 1.1408122843561244e-05, "spike_contrast": 3.0800260655612144, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.513731571729295e-05, "top_five_layers": [ 22, 21, 19, 23, 20 ], "top_five_spike_overlap": 3 }, "values": [ 8.121991413645446e-06, 1.4359444321598858e-05, 1.4293278582044877e-05, 9.803201464819722e-06, 8.053018973441795e-06, 1.3568884241976775e-05, 1.2741595128318295e-05, 7.793296390445903e-06, 7.450001248798799e-06, 9.841407518251799e-06, 1.0202589692198671e-05, 7.398411071335431e-06, 5.858043550688308e-06, 6.275422038015677e-06, 4.660452304960927e-06, 1.5999050447135232e-05, 1.6492198483319953e-05, 2.57461542787496e-05, 4.515285763773136e-05, 3.399932029424235e-05, 4.857672320213169e-05, 4.984432234778069e-05, 4.3853538954863325e-05, 2.2293334041023627e-05, 1.1118660040665418e-05, 7.616054062964395e-06, 9.428513294551522e-06, 4.200695002509747e-06, 3.730752496267087e-06, 1.991047156479908e-06, 1.8263393712913967e-06, 1.4152963103697402e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.0552482662169496e-05, "first_to_last_ratio": 1.9896078568885895, "imbalance_abs_log_ratio": 0.6879375624782461, "last_quartile_mean": 2.5408264491488808e-05, "mean": 6.979706753895698e-05, "normalized": [ 0.8424618714536765, 0.8875294097392207, 0.9051459937655827, 0.5457016616112889, 0.4773159928049252, 0.8683162175637178, 0.7704381752382723, 0.49731492759757023, 0.4420323183113381, 0.5165818577921957, 0.5794772315022629, 0.5076225155564954, 0.3309561455182541, 0.3052740112561091, 0.21389119365177953, 1.0441274181744666, 1.0008017526930533, 1.5667387755322346, 2.8174410811482873, 2.521932013314648, 4.568512624917261, 3.0003142184563067, 2.5420749618287632, 1.335753275629179, 0.7306479606172737, 0.4947636486838804, 0.6079571307037338, 0.3517508043350849, 0.29228869585353145, 0.17765058107663878, 0.15820247401055815, 0.09898305966240892 ], "peak_layer": 21, "peak_normalized": 4.568512624917261, "population_cv": 1.0020196982520382, "rest_mean": 5.1243189571121344e-05, "spike_contrast": 3.317280012973041, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001699880085652694, "top_five_layers": [ 21, 22, 19, 23, 20 ], "top_five_spike_overlap": 3 }, "values": [ 5.8801368140848354e-05, 6.194695015437901e-05, 6.317653605947271e-05, 3.8088375731604174e-05, 3.331525658722967e-05, 6.060592568246648e-05, 5.377432535169646e-05, 3.471112358965911e-05, 3.08525595755782e-05, 3.6055898817721754e-05, 4.0445811464451253e-05, 3.543056300259195e-05, 2.3099768441170454e-05, 2.130723078153096e-05, 1.4928978089301381e-05, 7.287703192560002e-05, 6.985302752582356e-05, 0.00010935377213172615, 0.000196649125427939, 0.00017602345906198025, 0.0003188687842339277, 0.00020941313414368778, 0.00017742937779985368, 9.323166159447283e-05, 5.0997085054405034e-05, 3.4533051803009585e-05, 4.243362491251901e-05, 2.455117464705836e-05, 2.040089384536259e-05, 1.239948960574111e-05, 1.1042068763345014e-05, 6.908727300469764e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "f4a14b64dd8b186b4b509748e2f608d960603b5d908ec49d21001a2020554f02", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04748110845685005, 0.06085919216275215, 0.06251973658800125, 0.08963073790073395, 0.08407715708017349, 0.1004372090101242, 0.09022688120603561, 0.09703134745359421, 0.0712415874004364, 0.07720176130533218, 0.07461326569318771, 0.0729745551943779, 0.06531906127929688, 0.06964895129203796, 0.08001933246850967, 0.11749708652496338, 0.09965827316045761, 0.0854005292057991, 0.08426972478628159, 0.0886540412902832, 0.04304825887084007, 0.0459144227206707, 0.05584317073225975, 0.06311307847499847, 0.05157631263136864, 0.05562245845794678, 0.059894196689128876, 0.06387592852115631, 0.050557736307382584, 0.06106090545654297, 0.06874485313892365, 0.07015988230705261 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.6830450085290067e-06, "first_to_last_ratio": 0.27205073217266473, "imbalance_abs_log_ratio": 1.3017667147372842, "last_quartile_mean": 1.3538081589103967e-05, "mean": 9.997345500778465e-06, "normalized": [ 0.8238008730306102, 0.43129250962679994, 0.49934591673609946, 0.29782217912043774, 0.18368099072454586, 0.1933449363361496, 0.23685570791400323, 0.28107523221499575, 0.3402086077396193, 0.32325728859883013, 0.37302383835188735, 0.43266448277670955, 0.41282186281653577, 0.3837163089578277, 0.40821318475885754, 0.2906458388933851, 0.44392610708725316, 0.5119919320955622, 0.5900289206272344, 0.8323966973801565, 3.728319067589544, 3.5968610962814633, 2.95823520549195, 2.5931302340463516, 2.1674970568044167, 1.890620302598499, 1.7758337928353976, 1.5730683335489977, 1.0522330833730802, 0.8211197895016011, 0.7910939465119015, 0.7618746756292967 ], "peak_layer": 21, "peak_normalized": 3.728319067589544, "population_cv": 0.9878337467468244, "rest_mean": 6.278317178603848e-06, "spike_contrast": 4.791108442726131, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0080098440521397e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 8.235821951529942e-06, 4.311780230636941e-06, 4.992133654013742e-06, 2.977431222461746e-06, 1.8363223261985695e-06, 1.932936129378504e-06, 2.3679283458477585e-06, 2.8100062081648502e-06, 3.401182993911789e-06, 3.23171479976736e-06, 3.7292481920303544e-06, 4.325496320234379e-06, 4.127122792851878e-06, 3.836144514934858e-06, 4.081048246007413e-06, 2.905686869780766e-06, 4.43808266936685e-06, 5.118560238770442e-06, 5.898722974961856e-06, 8.321757377416361e-06, 3.727329385583289e-05, 3.595906309783459e-05, 2.9574499421869405e-05, 2.59244188782759e-05, 2.16692169487942e-05, 1.8901184375863522e-05, 1.775362397893332e-05, 1.572650762682315e-05, 1.0519537681830116e-05, 8.209018233173992e-06, 7.908839506853838e-06, 7.616724360559601e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.4919307147920335e-05, "first_to_last_ratio": 0.2771847156933817, "imbalance_abs_log_ratio": 1.2830711513778186, "last_quartile_mean": 5.382442213885952e-05, "mean": 4.0030962580317464e-05, "normalized": [ 0.8279328105716447, 0.4385340769193414, 0.5113323447021809, 0.301967722777521, 0.17936700215896317, 0.19494385438915096, 0.241089047282958, 0.28638665602796354, 0.34043590168215715, 0.32758048161579845, 0.37911369328600636, 0.4350549514201117, 0.4161353264561751, 0.3865564890531755, 0.42474039805833697, 0.30722739751123224, 0.43933917507773224, 0.529530389422566, 0.5935744529609681, 0.8334989751506177, 3.7379165275353725, 3.5870637252031274, 2.9579615328299957, 2.566158910032566, 2.155837666405289, 1.8800235241322139, 1.7708849666906623, 1.5532705423788409, 1.044510485838323, 0.8147842371714528, 0.7815953628912844, 0.7556513723662709 ], "peak_layer": 21, "peak_normalized": 3.7379165275353725, "population_cv": 0.9835546199867091, "rest_mean": 2.519735838817454e-05, "spike_contrast": 4.767659504905444, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012013242521788925, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.314294735901058e-05, 1.7554941223352216e-05, 2.0469125956878997e-05, 1.208805861097062e-05, 7.180233751569176e-06, 7.803790140314959e-06, 9.651026630308479e-06, 1.1464333510957658e-05, 1.3627976841235068e-05, 1.31133620016044e-05, 1.5176286069618072e-05, 1.7415668480680324e-05, 1.665829768171534e-05, 1.5474228348466568e-05, 1.7002766981022432e-05, 1.2298608453420456e-05, 1.758717007760424e-05, 2.1197611204115674e-05, 2.3761356715112925e-05, 3.336576628498733e-05, 0.0001496323966421187, 0.00014359361375682056, 0.00011841004743473604, 0.0001027258113026619, 8.630025695310906e-05, 7.525915134465322e-05, 7.089022983564064e-05, 6.217891495907679e-05, 4.181276017334312e-05, 3.2616597309242934e-05, 3.1288014724850655e-05, 3.0249451810959727e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.013230590906459838, "first_to_last_ratio": 0.27718471002181044, "imbalance_abs_log_ratio": 1.2830711718391608, "last_quartile_mean": 0.04773203725926578, "mean": 0.035499858771800064, "normalized": [ 0.8279327972123818, 0.43853407753129525, 0.5113323373845421, 0.3019677236670519, 0.17936700088464805, 0.19494385266486466, 0.24108904976283277, 0.28638663500812633, 0.3404359064207765, 0.3275804556161215, 0.37911367693924214, 0.43505493113041177, 0.4161352869870187, 0.3865564955173268, 0.4247403745275919, 0.30722738227286805, 0.43933918942416506, 0.529530400420262, 0.5935744293554726, 0.8334989802196542, 3.7379165882450245, 3.58706376795423, 2.95796152343449, 2.566158906335581, 2.1558376400844, 1.8800236046449208, 1.7708849671242992, 1.5532705865178382, 1.044510493614123, 0.814784201623246, 0.7815953533826546, 0.7556513840925386 ], "peak_layer": 21, "peak_normalized": 3.7379165882450245, "population_cv": 0.9835546333023998, "rest_mean": 0.022345269840486622, "spike_contrast": 4.767659543223157, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10653463900089263, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.029391497373580933, 0.015567897818982601, 0.018152225762605667, 0.010719811543822289, 0.006367503199726343, 0.006920479238033295, 0.008558627218008041, 0.010166685096919537, 0.012085426598787308, 0.011629059910774231, 0.01345848198980093, 0.01544438861310482, 0.014772743918001652, 0.013722700998187065, 0.015078223310410976, 0.010906528681516647, 0.015596479177474976, 0.018798254430294037, 0.02107180841267109, 0.0295890960842371, 0.1326955109834671, 0.12734025716781616, 0.10500721633434296, 0.09109827876091003, 0.07653193175792694, 0.06674057245254517, 0.06286616623401642, 0.05514088645577431, 0.03707997500896454, 0.028924724087119102, 0.027746524661779404, 0.026825517416000366 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.408127326082649e-07, "first_to_last_ratio": 0.26109600872884425, "imbalance_abs_log_ratio": 1.3428670897636954, "last_quartile_mean": 3.6033210051300557e-06, "mean": 2.51254786931554e-06, "normalized": [ 0.9411291774986253, 0.42452308852469994, 0.5064277702253434, 0.31422637836100054, 0.14909909853668837, 0.18364790421678406, 0.20923427416444018, 0.2672778667582645, 0.28101884819319983, 0.26433726849666683, 0.32343818509729044, 0.3930866637355723, 0.38182199462519517, 0.3573030049702274, 0.36652953981026026, 0.20759778048664343, 0.36120097522274675, 0.33747924881801133, 0.4694224480590536, 0.6406335998556872, 3.722312978049966, 3.6582343696129027, 3.0403294690381584, 2.726645745510304, 2.2400013253030986, 1.9540567521771424, 1.8942350408219104, 1.6614529947683714, 1.1379024352010345, 0.8900222502819433, 0.8554212967076426, 0.8399502268711246 ], "peak_layer": 21, "peak_normalized": 3.722312978049966, "population_cv": 1.0300414744999016, "rest_mean": 1.5459126466844787e-06, "spike_contrast": 5.0018208261035415, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.732378071523271e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.364632109674858e-06, 1.0666345815479872e-06, 1.2724240150419064e-06, 7.895088174336706e-07, 3.7461862234522414e-07, 4.614241504441452e-07, 5.257111297396477e-07, 6.715484346386802e-07, 7.060733082653314e-07, 6.641600407419901e-07, 8.126539228214824e-07, 9.876490594251663e-07, 9.593460390533437e-07, 8.977409038379847e-07, 9.209230142914748e-07, 5.215993610363512e-07, 9.075347406906076e-07, 8.479327675559034e-07, 1.17944637167966e-06, 1.6096225863293512e-06, 9.352489541925024e-06, 9.191488970827777e-06, 7.638973329449072e-06, 6.850827958260197e-06, 5.628110557154287e-06, 4.9096611292043235e-06, 4.759356215799926e-06, 4.174480181973195e-06, 2.8590343390533235e-06, 2.236223508589319e-06, 2.1492869564099237e-06, 2.1104151528561488e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.0381400045389455e-05, "first_to_last_ratio": 0.30181441355404504, "imbalance_abs_log_ratio": 1.1979429751604156, "last_quartile_mean": 3.439663441895391e-05, "mean": 2.6535070020372586e-05, "normalized": [ 0.9040898455603302, 0.4673694653321381, 0.5438132272984187, 0.2908698623251208, 0.18839899103808294, 0.1956218714505659, 0.25253197694418594, 0.28717026631709835, 0.3611321614066451, 0.34954594583315046, 0.4132621282293166, 0.48655346340205047, 0.47102858233475486, 0.4382954282683324, 0.4460853575084453, 0.28452002254446673, 0.469204525002259, 0.546001458578785, 0.6236410135514943, 0.8373328706810375, 3.7465220405244266, 3.619599622084698, 2.9108020234237175, 2.49644206069555, 2.1762350991842796, 1.8187600325437407, 1.7041392558246975, 1.4815869865618754, 0.9869504724512707, 0.7532641117134347, 0.7324670886872183, 0.7167627426984122 ], "peak_layer": 21, "peak_normalized": 3.7465220405244266, "population_cv": 0.9710857842550003, "rest_mean": 1.6756797608481805e-05, "spike_contrast": 4.734660100234446, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.93377410445828e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.3990087356651202e-05, 1.2401681487972382e-05, 1.4430122064368334e-05, 7.718252163613215e-06, 4.999180418963078e-06, 5.190840056457091e-06, 6.700953690597089e-06, 7.620083124493249e-06, 9.582667189533822e-06, 9.27522614802001e-06, 1.0965939509333111e-05, 1.29107302200282e-05, 1.2498776413849555e-05, 1.163019987870939e-05, 1.1836906196549535e-05, 7.5497587204154115e-06, 1.2450374924810603e-05, 1.4488186934613623e-05, 1.6548357962165028e-05, 2.2218686353880912e-05, 9.941422467818484e-05, 9.604632941773161e-05, 7.723833550699055e-05, 6.624326488235965e-05, 5.774655073764734e-05, 4.8260924813803285e-05, 4.521945447777398e-05, 3.931401442969218e-05, 2.6188799893134274e-05, 1.9987915948149748e-05, 1.9436065485933796e-05, 1.9019349565496668e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 7.0662427162915264e-06, "first_to_last_ratio": 0.31298041774626, "imbalance_abs_log_ratio": 1.1616146535101273, "last_quartile_mean": 2.2577267827728065e-05, "mean": 1.7418877341413008e-05, "normalized": [ 0.9686515201468897, 0.49383920743252663, 0.5592845449086765, 0.2807880642348226, 0.19794654052965874, 0.19434619484963173, 0.2653113252926559, 0.28515895873794617, 0.3736472447718833, 0.36301406000122344, 0.43599250394692873, 0.5281282188816567, 0.5269171867525786, 0.48757686495182617, 0.47960088642402976, 0.274044916832133, 0.5050358505640102, 0.5616893266095021, 0.6389811522580301, 0.8090338932687061, 3.6651836739778836, 3.5584695736966103, 2.7908959611190953, 2.3873587945098413, 2.235209216498694, 1.7539959875660012, 1.6837970592001588, 1.45698578925774, 1.0345422359131593, 0.7653898492004969, 0.7229649825149719, 0.716218415150031 ], "peak_layer": 21, "peak_normalized": 3.6651836739778836, "population_cv": 0.9404784507863737, "rest_mean": 1.1201552794133292e-05, "spike_contrast": 4.552264389936214, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.099242989672348e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.6872822016011924e-05, 8.602124580647796e-06, 9.74210888671223e-06, 4.891012849839171e-06, 3.4480065096431645e-06, 3.3852925298560876e-06, 4.6214254325605e-06, 4.967148925061338e-06, 6.508515525638359e-06, 6.323297384369653e-06, 7.594499948027078e-06, 9.1994006652385e-06, 9.178305845125578e-06, 8.493041605106555e-06, 8.354109013453126e-06, 4.773554792336654e-06, 8.797157533990685e-06, 9.783997484191786e-06, 1.1130334314657375e-05, 1.4092462151893415e-05, 6.384338485077024e-05, 6.19845450273715e-05, 4.861427441937849e-05, 4.158511001151055e-05, 3.8934835174586624e-05, 3.055264096474275e-05, 2.9329854442039505e-05, 2.5379056751262397e-05, 1.802056431188248e-05, 1.3332231901586056e-05, 1.2593238352565095e-05, 1.2475720723159611e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.965646217489848e-05, "first_to_last_ratio": 0.25778316490635544, "imbalance_abs_log_ratio": 1.3556364935327478, "last_quartile_mean": 0.00011504421627250849, "mean": 8.427746251982171e-05, "normalized": [ 0.7921668867268374, 0.4122773826527116, 0.48829879548757044, 0.2751554675194965, 0.1662779686404814, 0.1822655064180769, 0.22566214189861117, 0.2730221051205942, 0.330970122717494, 0.3135140758471058, 0.3728659820070459, 0.42617092604953055, 0.39288617322467556, 0.3613498654548339, 0.3791111655490991, 0.2670862108876096, 0.42548587073411964, 0.4864503548031842, 0.5713051825384278, 0.8003801258416829, 3.770712978765767, 3.65288415518208, 3.0183549283675815, 2.6948254474092166, 2.183148265157847, 1.929015794151022, 1.9467070562963738, 1.6704513131036254, 0.9796587997179057, 0.7674221880006324, 0.7441547396212447, 0.6999620241075153 ], "peak_layer": 21, "peak_normalized": 3.770712978765767, "population_cv": 1.0201901284069992, "rest_mean": 5.206497519759944e-05, "spike_contrast": 4.95966613024965, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000258224894059822, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.676181510556489e-05, 3.4745691664284095e-05, 4.11525834351778e-05, 2.3189404600998387e-05, 1.401348526997026e-05, 1.53608743858058e-05, 1.901823270600289e-05, 2.3009610231383704e-05, 2.7893322112504393e-05, 2.6422170776640996e-05, 3.1424198823515326e-05, 3.591660424717702e-05, 3.311144973849878e-05, 3.045364974241238e-05, 3.1950527045410126e-05, 2.2509348127641715e-05, 3.5858869523508474e-05, 4.099680154467933e-05, 4.814815110876225e-05, 6.745400605723262e-05, 0.00031778612174093723, 0.0003078558074776083, 0.000254379294347018, 0.000227113050641492, 0.00018399019609205425, 0.00016257255629170686, 0.0001640635309740901, 0.00014078139793127775, 8.256315777543932e-05, 6.467639468610287e-05, 6.271547317737713e-05, 5.899102325201966e-05 ] } } } } } }, "step": 2000 }, { "modes": { "learned": { "bits_per_byte": 1.9288425276664853, "forward": { "activation_sha256": { "attention_branch_output": "41db358e51c17755a85e4f4c84288ab48da8d214d2518c64ab3d3ea8eb9826e1", "mlp_branch_output": "6c58c506475022833bc648a96c247244a57dd96d762e84fedda83974a9fa4cd0", "post_attention_state": "90bf4c661b8e06ef22fe9277f79a81ef01617138f2a24fe93a0093039186433f", "post_mlp_state": "25421e314b847c78a706fbfc95b3ae6bf0cea5ddc086f201de56fb1ddc434ca3", "pre_attention_input": "36b6c956f67457ef008d5eacc14cf9477822b0791655ecd816dc4fb89c68ffd5", "pre_mlp_input": "c1e7ba7b2fabbdfa228a4a3c33df9fd2cb9236dae2c5351a1ecf233f5b8dce87" }, "logits_sha256": "562529f2e73d225129c831e3649b7f416f6e2834513ba88186693cdef5a65cc3", "loss_nats": 1.3369717597961426, "mixer_summary_sha256": "900263e7a76f1202bf5b394a28abd810902e027421616ad02b8950572b564b66" }, "loss_nats": 1.3369717597961426, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.2508508265018463, "group": 1, "latest_source_mass": 0.07481259852647781, "layer": 1, "max_source_mass": 0.9251874089241028, "mixer_index": 1, "normalized_entropy": 0.36190124339711144, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9251874089241028, "median": 0.939024806022644, "p05": 0.8262566328048706, "p95": 0.9732407331466675 }, { "label": "current_group_partial", "mean": 0.07481259852647781, "median": 0.06097518280148506, "p05": 0.026759283617138863, "p95": 0.1737433522939682 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5796181559562683, "group": 1, "latest_source_mass": 0.3836008310317993, "layer": 2, "max_source_mass": 0.6163992285728455, "mixer_index": 2, "normalized_entropy": 0.8362122392073141, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.6163992285728455, "median": 0.6123820543289185, "p05": 0.30569732189178467, "p95": 0.9099069833755493 }, { "label": "current_group_partial", "mean": 0.3836008310317993, "median": 0.38761788606643677, "p05": 0.09009299427270889, "p95": 0.6943026781082153 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3817020058631897, "group": 1, "latest_source_mass": 0.13486965000629425, "layer": 2, "max_source_mass": 0.8651303648948669, "mixer_index": 3, "normalized_entropy": 0.5506795909561938, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8651303648948669, "median": 0.882187008857727, "p05": 0.7400804758071899, "p95": 0.931220531463623 }, { "label": "current_group_partial", "mean": 0.13486965000629425, "median": 0.11781297624111176, "p05": 0.06877946853637695, "p95": 0.25991958379745483 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6884468793869019, "group": 1, "latest_source_mass": 0.5067460536956787, "layer": 3, "max_source_mass": 0.5067460536956787, "mixer_index": 4, "normalized_entropy": 0.9932188988069657, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4932539165019989, "median": 0.4924398362636566, "p05": 0.4183998703956604, "p95": 0.5739172101020813 }, { "label": "current_group_partial", "mean": 0.5067460536956787, "median": 0.507560133934021, "p05": 0.4260827600955963, "p95": 0.5816001892089844 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.2797870337963104, "group": 1, "latest_source_mass": 0.08572537451982498, "layer": 3, "max_source_mass": 0.9142746329307556, "mixer_index": 5, "normalized_entropy": 0.40364736616296987, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.9142746329307556, "median": 0.9290047883987427, "p05": 0.8227594494819641, "p95": 0.965082049369812 }, { "label": "current_group_partial", "mean": 0.08572537451982498, "median": 0.07099512964487076, "p05": 0.03491789847612381, "p95": 0.1772405207157135 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6642253994941711, "group": 1, "latest_source_mass": 0.39495500922203064, "layer": 4, "max_source_mass": 0.605044960975647, "mixer_index": 6, "normalized_entropy": 0.9582746898827313, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.605044960975647, "median": 0.6064984202384949, "p05": 0.5077860951423645, "p95": 0.6980465054512024 }, { "label": "current_group_partial", "mean": 0.39495500922203064, "median": 0.39350154995918274, "p05": 0.3019534945487976, "p95": 0.4922138750553131 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5038842558860779, "group": 1, "latest_source_mass": 0.2243884801864624, "layer": 4, "max_source_mass": 0.7756115198135376, "mixer_index": 7, "normalized_entropy": 0.7269513171488701, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7756115198135376, "median": 0.786786675453186, "p05": 0.6057990789413452, "p95": 0.9406262636184692 }, { "label": "current_group_partial", "mean": 0.2243884801864624, "median": 0.21321335434913635, "p05": 0.059373728930950165, "p95": 0.3942009508609772 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6434851884841919, "group": 2, "latest_source_mass": 0.4625895023345947, "layer": 5, "max_source_mass": 0.5374104976654053, "mixer_index": 8, "normalized_entropy": 0.9283528903116436, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5374104976654053, "median": 0.522662878036499, "p05": 0.32466351985931396, "p95": 0.7907323837280273 }, { "label": "completed_group_1", "mean": 0.4625895023345947, "median": 0.47733718156814575, "p05": 0.20926763117313385, "p95": 0.675336480140686 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0016276836395264, "group": 2, "latest_source_mass": 0.37876495718955994, "layer": 5, "max_source_mass": 0.4353652000427246, "mixer_index": 9, "normalized_entropy": 0.911720808124073, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4353652000427246, "median": 0.43071597814559937, "p05": 0.24926689267158508, "p95": 0.665458083152771 }, { "label": "completed_group_1", "mean": 0.18586981296539307, "median": 0.18826425075531006, "p05": 0.1286066770553589, "p95": 0.24030350148677826 }, { "label": "current_group_partial", "mean": 0.37876495718955994, "median": 0.3880766034126282, "p05": 0.176315575838089, "p95": 0.5597050189971924 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0655895471572876, "group": 2, "latest_source_mass": 0.4483259618282318, "layer": 6, "max_source_mass": 0.4483259618282318, "mixer_index": 10, "normalized_entropy": 0.9699414053060913, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2726097106933594, "median": 0.2658982276916504, "p05": 0.22330613434314728, "p95": 0.3535451292991638 }, { "label": "completed_group_1", "mean": 0.2790643572807312, "median": 0.2796473205089569, "p05": 0.2485380917787552, "p95": 0.30792567133903503 }, { "label": "current_group_partial", "mean": 0.4483259618282318, "median": 0.4539736807346344, "p05": 0.378553181886673, "p95": 0.4919973611831665 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0300546884536743, "group": 2, "latest_source_mass": 0.3101818859577179, "layer": 6, "max_source_mass": 0.40803641080856323, "mixer_index": 11, "normalized_entropy": 0.9375961830014203, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.40803641080856323, "median": 0.39597707986831665, "p05": 0.17321011424064636, "p95": 0.7176744341850281 }, { "label": "completed_group_1", "mean": 0.2817816734313965, "median": 0.2903806269168854, "p05": 0.1267791986465454, "p95": 0.37898188829421997 }, { "label": "current_group_partial", "mean": 0.3101818859577179, "median": 0.312465637922287, "p05": 0.13594681024551392, "p95": 0.466165691614151 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0874333381652832, "group": 2, "latest_source_mass": 0.35985422134399414, "layer": 7, "max_source_mass": 0.36608952283859253, "mixer_index": 12, "normalized_entropy": 0.9898244807398074, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.36608952283859253, "median": 0.37737005949020386, "p05": 0.2977083623409271, "p95": 0.4045135974884033 }, { "label": "completed_group_1", "mean": 0.27405625581741333, "median": 0.2681817412376404, "p05": 0.25118786096572876, "p95": 0.3135649561882019 }, { "label": "current_group_partial", "mean": 0.35985422134399414, "median": 0.35611432790756226, "p05": 0.33197838068008423, "p95": 0.3942449688911438 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9672750234603882, "group": 2, "latest_source_mass": 0.2810039520263672, "layer": 7, "max_source_mass": 0.5343947410583496, "mixer_index": 13, "normalized_entropy": 0.8804516692900397, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5343947410583496, "median": 0.5141075849533081, "p05": 0.3664405345916748, "p95": 0.8041804432868958 }, { "label": "completed_group_1", "mean": 0.1846013069152832, "median": 0.18730750679969788, "p05": 0.07676203548908234, "p95": 0.26666438579559326 }, { "label": "current_group_partial", "mean": 0.2810039520263672, "median": 0.2932106554508209, "p05": 0.11621598154306412, "p95": 0.3943445682525635 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0731852054595947, "group": 2, "latest_source_mass": 0.37608763575553894, "layer": 8, "max_source_mass": 0.3801807165145874, "mixer_index": 14, "normalized_entropy": 0.976855271444905, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3801807165145874, "median": 0.39405983686447144, "p05": 0.2808566689491272, "p95": 0.44335222244262695 }, { "label": "completed_group_1", "mean": 0.24373164772987366, "median": 0.2364288568496704, "p05": 0.20917385816574097, "p95": 0.29448947310447693 }, { "label": "current_group_partial", "mean": 0.37608763575553894, "median": 0.3712879717350006, "p05": 0.3317960500717163, "p95": 0.4302467107772827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9721636772155762, "group": 2, "latest_source_mass": 0.29772043228149414, "layer": 8, "max_source_mass": 0.49137914180755615, "mixer_index": 15, "normalized_entropy": 0.8849015137034084, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.49137914180755615, "median": 0.4575059711933136, "p05": 0.2624281048774719, "p95": 0.8368043899536133 }, { "label": "completed_group_1", "mean": 0.2109004259109497, "median": 0.20731960237026215, "p05": 0.05826364830136299, "p95": 0.34210139513015747 }, { "label": "current_group_partial", "mean": 0.29772043228149414, "median": 0.3127709627151489, "p05": 0.10320040583610535, "p95": 0.44153448939323425 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0864832401275635, "group": 3, "latest_source_mass": 0.3128502070903778, "layer": 9, "max_source_mass": 0.39626434445381165, "mixer_index": 16, "normalized_entropy": 0.9889596642367338, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.39626434445381165, "median": 0.3972475528717041, "p05": 0.3466483950614929, "p95": 0.44265681505203247 }, { "label": "completed_group_1", "mean": 0.29088544845581055, "median": 0.289419949054718, "p05": 0.26393142342567444, "p95": 0.3211052417755127 }, { "label": "completed_group_2", "mean": 0.3128502070903778, "median": 0.31452691555023193, "p05": 0.2603488564491272, "p95": 0.36055514216423035 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3038380146026611, "group": 3, "latest_source_mass": 0.29599112272262573, "layer": 9, "max_source_mass": 0.35696184635162354, "mixer_index": 17, "normalized_entropy": 0.9405203188948855, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.35696184635162354, "median": 0.36454010009765625, "p05": 0.19070445001125336, "p95": 0.5144885778427124 }, { "label": "completed_group_1", "mean": 0.1472446471452713, "median": 0.14020296931266785, "p05": 0.08932732790708542, "p95": 0.22387003898620605 }, { "label": "completed_group_2", "mean": 0.19980241358280182, "median": 0.1939706951379776, "p05": 0.11784223467111588, "p95": 0.2938849925994873 }, { "label": "current_group_partial", "mean": 0.29599112272262573, "median": 0.2978004813194275, "p05": 0.23555772006511688, "p95": 0.3528674244880676 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3611116409301758, "group": 3, "latest_source_mass": 0.2453548014163971, "layer": 10, "max_source_mass": 0.3264957070350647, "mixer_index": 18, "normalized_entropy": 0.9818345072331021, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3264957070350647, "median": 0.33824437856674194, "p05": 0.23342323303222656, "p95": 0.3961111009120941 }, { "label": "completed_group_1", "mean": 0.22676822543144226, "median": 0.225631982088089, "p05": 0.20189295709133148, "p95": 0.25452396273612976 }, { "label": "completed_group_2", "mean": 0.20138128101825714, "median": 0.1983025223016739, "p05": 0.17473457753658295, "p95": 0.23882760107517242 }, { "label": "current_group_partial", "mean": 0.2453548014163971, "median": 0.24060846865177155, "p05": 0.1962868720293045, "p95": 0.3133445978164673 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3161137104034424, "group": 3, "latest_source_mass": 0.2900698184967041, "layer": 10, "max_source_mass": 0.3259783387184143, "mixer_index": 19, "normalized_entropy": 0.9493753616225099, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3259783387184143, "median": 0.35052698850631714, "p05": 0.10893625020980835, "p95": 0.48943614959716797 }, { "label": "completed_group_1", "mean": 0.17862649261951447, "median": 0.18309253454208374, "p05": 0.1213371679186821, "p95": 0.21994648873806 }, { "label": "completed_group_2", "mean": 0.20532533526420593, "median": 0.19941335916519165, "p05": 0.12927237153053284, "p95": 0.29834914207458496 }, { "label": "current_group_partial", "mean": 0.2900698184967041, "median": 0.2715068459510803, "p05": 0.20594194531440735, "p95": 0.42361485958099365 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3568220138549805, "group": 3, "latest_source_mass": 0.27865856885910034, "layer": 11, "max_source_mass": 0.3223523199558258, "mixer_index": 20, "normalized_entropy": 0.9787401953787784, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3223523199558258, "median": 0.31804126501083374, "p05": 0.2675665020942688, "p95": 0.3895469605922699 }, { "label": "completed_group_1", "mean": 0.21851874887943268, "median": 0.21804103255271912, "p05": 0.18837261199951172, "p95": 0.251605361700058 }, { "label": "completed_group_2", "mean": 0.1804703176021576, "median": 0.1816348135471344, "p05": 0.15208400785923004, "p95": 0.2051190286874771 }, { "label": "current_group_partial", "mean": 0.27865856885910034, "median": 0.2795049548149109, "p05": 0.22788430750370026, "p95": 0.3272843062877655 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2413711547851562, "group": 3, "latest_source_mass": 0.26951953768730164, "layer": 11, "max_source_mass": 0.4272037148475647, "mixer_index": 21, "normalized_entropy": 0.8954600044555754, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4272037148475647, "median": 0.4580039381980896, "p05": 0.18249742686748505, "p95": 0.6239475607872009 }, { "label": "completed_group_1", "mean": 0.16667886078357697, "median": 0.1573459506034851, "p05": 0.0859292671084404, "p95": 0.26250067353248596 }, { "label": "completed_group_2", "mean": 0.1365978717803955, "median": 0.13323482871055603, "p05": 0.07157228887081146, "p95": 0.2005852907896042 }, { "label": "current_group_partial", "mean": 0.26951953768730164, "median": 0.2574368417263031, "p05": 0.1767870932817459, "p95": 0.40349671244621277 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3540384769439697, "group": 3, "latest_source_mass": 0.2906486392021179, "layer": 12, "max_source_mass": 0.32447606325149536, "mixer_index": 22, "normalized_entropy": 0.9767322979299551, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.32447606325149536, "median": 0.31636133790016174, "p05": 0.2827298045158386, "p95": 0.3818347454071045 }, { "label": "completed_group_1", "mean": 0.2030642032623291, "median": 0.20119589567184448, "p05": 0.18187163770198822, "p95": 0.22999082505702972 }, { "label": "completed_group_2", "mean": 0.18181107938289642, "median": 0.18250997364521027, "p05": 0.15624107420444489, "p95": 0.2048231065273285 }, { "label": "current_group_partial", "mean": 0.2906486392021179, "median": 0.2923159897327423, "p05": 0.24348178505897522, "p95": 0.33497530221939087 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.1544246673583984, "group": 3, "latest_source_mass": 0.22926729917526245, "layer": 12, "max_source_mass": 0.5128031969070435, "mixer_index": 23, "normalized_entropy": 0.8327413713389263, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5128031969070435, "median": 0.4827004075050354, "p05": 0.32023805379867554, "p95": 0.7668037414550781 }, { "label": "completed_group_1", "mean": 0.13654223084449768, "median": 0.13301728665828705, "p05": 0.05933574587106705, "p95": 0.21417926251888275 }, { "label": "completed_group_2", "mean": 0.12138727307319641, "median": 0.12862378358840942, "p05": 0.05767478048801422, "p95": 0.1747671365737915 }, { "label": "current_group_partial", "mean": 0.22926729917526245, "median": 0.22523194551467896, "p05": 0.10545835644006729, "p95": 0.37030228972435 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3712997436523438, "group": 4, "latest_source_mass": 0.22271712124347687, "layer": 13, "max_source_mass": 0.31037354469299316, "mixer_index": 24, "normalized_entropy": 0.9891836698697716, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.31037354469299316, "median": 0.3152464032173157, "p05": 0.24687863886356354, "p95": 0.3615240454673767 }, { "label": "completed_group_1", "mean": 0.24219870567321777, "median": 0.24224354326725006, "p05": 0.20525576174259186, "p95": 0.2801668643951416 }, { "label": "completed_group_2", "mean": 0.2247106432914734, "median": 0.2232801616191864, "p05": 0.19794036448001862, "p95": 0.25516608357429504 }, { "label": "completed_group_3", "mean": 0.22271712124347687, "median": 0.22098374366760254, "p05": 0.185325026512146, "p95": 0.2633301615715027 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5205519199371338, "group": 4, "latest_source_mass": 0.2427731454372406, "layer": 13, "max_source_mass": 0.330753892660141, "mixer_index": 25, "normalized_entropy": 0.9447720276686312, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.330753892660141, "median": 0.3369896411895752, "p05": 0.22567971050739288, "p95": 0.46019247174263 }, { "label": "completed_group_1", "mean": 0.1283297836780548, "median": 0.13036663830280304, "p05": 0.09653771668672562, "p95": 0.15339873731136322 }, { "label": "completed_group_2", "mean": 0.12636876106262207, "median": 0.1281418800354004, "p05": 0.095677949488163, "p95": 0.1535145789384842 }, { "label": "completed_group_3", "mean": 0.17177443206310272, "median": 0.16995373368263245, "p05": 0.12080889940261841, "p95": 0.22754061222076416 }, { "label": "current_group_partial", "mean": 0.2427731454372406, "median": 0.2397468388080597, "p05": 0.19260814785957336, "p95": 0.3004864454269409 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.518624186515808, "group": 4, "latest_source_mass": 0.3593595027923584, "layer": 14, "max_source_mass": 0.3593595027923584, "mixer_index": 26, "normalized_entropy": 0.9435742595494434, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2150207757949829, "median": 0.21538454294204712, "p05": 0.16830593347549438, "p95": 0.263691782951355 }, { "label": "completed_group_1", "mean": 0.17106646299362183, "median": 0.17181023955345154, "p05": 0.13603076338768005, "p95": 0.20341315865516663 }, { "label": "completed_group_2", "mean": 0.13557285070419312, "median": 0.1350601464509964, "p05": 0.11946140229701996, "p95": 0.153931125998497 }, { "label": "completed_group_3", "mean": 0.11898040026426315, "median": 0.11543577164411545, "p05": 0.10078167170286179, "p95": 0.14709262549877167 }, { "label": "current_group_partial", "mean": 0.3593595027923584, "median": 0.36046332120895386, "p05": 0.3041363060474396, "p95": 0.40930092334747314 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4772696495056152, "group": 4, "latest_source_mass": 0.3085501194000244, "layer": 14, "max_source_mass": 0.3085501194000244, "mixer_index": 27, "normalized_entropy": 0.9178792410024722, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30324018001556396, "median": 0.3082737922668457, "p05": 0.16577009856700897, "p95": 0.4784274101257324 }, { "label": "completed_group_1", "mean": 0.13238272070884705, "median": 0.12282240390777588, "p05": 0.08379106968641281, "p95": 0.2037200629711151 }, { "label": "completed_group_2", "mean": 0.0979682207107544, "median": 0.09772157669067383, "p05": 0.0601615384221077, "p95": 0.13479545712471008 }, { "label": "completed_group_3", "mean": 0.15785875916481018, "median": 0.1573975682258606, "p05": 0.11552698910236359, "p95": 0.20258599519729614 }, { "label": "current_group_partial", "mean": 0.3085501194000244, "median": 0.3097953200340271, "p05": 0.21463434398174286, "p95": 0.40537965297698975 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4979676008224487, "group": 4, "latest_source_mass": 0.3820047378540039, "layer": 15, "max_source_mass": 0.3820047378540039, "mixer_index": 28, "normalized_entropy": 0.9307396012294349, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20932656526565552, "median": 0.20591703057289124, "p05": 0.1635243147611618, "p95": 0.2720984220504761 }, { "label": "completed_group_1", "mean": 0.1654297560453415, "median": 0.16333067417144775, "p05": 0.1427382081747055, "p95": 0.19703316688537598 }, { "label": "completed_group_2", "mean": 0.1272144615650177, "median": 0.12634585797786713, "p05": 0.10835681855678558, "p95": 0.14949949085712433 }, { "label": "completed_group_3", "mean": 0.1160244345664978, "median": 0.11398661881685257, "p05": 0.09442843496799469, "p95": 0.14416006207466125 }, { "label": "current_group_partial", "mean": 0.3820047378540039, "median": 0.38286471366882324, "p05": 0.31784284114837646, "p95": 0.439454048871994 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.49334716796875, "group": 4, "latest_source_mass": 0.31850871443748474, "layer": 15, "max_source_mass": 0.31850871443748474, "mixer_index": 29, "normalized_entropy": 0.927868764884645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.28671571612358093, "median": 0.27600669860839844, "p05": 0.15595941245555878, "p95": 0.4432004988193512 }, { "label": "completed_group_1", "mean": 0.13947689533233643, "median": 0.13652008771896362, "p05": 0.10096997767686844, "p95": 0.18479548394680023 }, { "label": "completed_group_2", "mean": 0.11353637278079987, "median": 0.11486998945474625, "p05": 0.07941771298646927, "p95": 0.14473722875118256 }, { "label": "completed_group_3", "mean": 0.14176231622695923, "median": 0.14077864587306976, "p05": 0.10397820174694061, "p95": 0.18330445885658264 }, { "label": "current_group_partial", "mean": 0.31850871443748474, "median": 0.31773626804351807, "p05": 0.22442427277565002, "p95": 0.41696855425834656 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5120060443878174, "group": 4, "latest_source_mass": 0.28145554661750793, "layer": 16, "max_source_mass": 0.28788846731185913, "mixer_index": 30, "normalized_entropy": 0.9394621766434421, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21571135520935059, "median": 0.21688061952590942, "p05": 0.16800610721111298, "p95": 0.2583240270614624 }, { "label": "completed_group_1", "mean": 0.28788846731185913, "median": 0.2848382592201233, "p05": 0.23045042157173157, "p95": 0.35751083493232727 }, { "label": "completed_group_2", "mean": 0.131511390209198, "median": 0.12990695238113403, "p05": 0.11185698211193085, "p95": 0.15568122267723083 }, { "label": "completed_group_3", "mean": 0.08343322575092316, "median": 0.0825304388999939, "p05": 0.05796157196164131, "p95": 0.1136971041560173 }, { "label": "current_group_partial", "mean": 0.28145554661750793, "median": 0.28277498483657837, "p05": 0.23862068355083466, "p95": 0.32226449251174927 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5766459703445435, "group": 4, "latest_source_mass": 0.20234227180480957, "layer": 16, "max_source_mass": 0.2565065026283264, "mixer_index": 31, "normalized_entropy": 0.9796252208077026, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2565065026283264, "median": 0.2449997365474701, "p05": 0.18619969487190247, "p95": 0.3817925453186035 }, { "label": "completed_group_1", "mean": 0.22934341430664062, "median": 0.2310539335012436, "p05": 0.19203530251979828, "p95": 0.2589260935783386 }, { "label": "completed_group_2", "mean": 0.1593727320432663, "median": 0.16161103546619415, "p05": 0.12202638387680054, "p95": 0.19153417646884918 }, { "label": "completed_group_3", "mean": 0.1524350941181183, "median": 0.14918631315231323, "p05": 0.12257804721593857, "p95": 0.19255484640598297 }, { "label": "current_group_partial", "mean": 0.20234227180480957, "median": 0.19873467087745667, "p05": 0.1501774936914444, "p95": 0.2577142119407654 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5796699523925781, "group": 5, "latest_source_mass": 0.2423015832901001, "layer": 17, "max_source_mass": 0.2423015832901001, "mixer_index": 32, "normalized_entropy": 0.9815041264956277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.23561140894889832, "median": 0.23130320012569427, "p05": 0.19288715720176697, "p95": 0.2910196781158447 }, { "label": "completed_group_1", "mean": 0.21223093569278717, "median": 0.20132887363433838, "p05": 0.16995927691459656, "p95": 0.29450756311416626 }, { "label": "completed_group_2", "mean": 0.14396421611309052, "median": 0.1432584673166275, "p05": 0.1268429160118103, "p95": 0.1634216159582138 }, { "label": "completed_group_3", "mean": 0.1658918410539627, "median": 0.16675393283367157, "p05": 0.1365433931350708, "p95": 0.19156640768051147 }, { "label": "completed_group_4", "mean": 0.2423015832901001, "median": 0.24500936269760132, "p05": 0.16469213366508484, "p95": 0.3121073544025421 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7626371383666992, "group": 5, "latest_source_mass": 0.18827059864997864, "layer": 17, "max_source_mass": 0.20886825025081635, "mixer_index": 33, "normalized_entropy": 0.983746517676336, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16389364004135132, "median": 0.16299191117286682, "p05": 0.12742109596729279, "p95": 0.20235949754714966 }, { "label": "completed_group_1", "mean": 0.20886825025081635, "median": 0.2163066267967224, "p05": 0.15428033471107483, "p95": 0.24192781746387482 }, { "label": "completed_group_2", "mean": 0.18027648329734802, "median": 0.18112054467201233, "p05": 0.15291966497898102, "p95": 0.20525163412094116 }, { "label": "completed_group_3", "mean": 0.1462823748588562, "median": 0.1460549235343933, "p05": 0.1107979416847229, "p95": 0.1837027370929718 }, { "label": "completed_group_4", "mean": 0.11240866780281067, "median": 0.11029648035764694, "p05": 0.09211881458759308, "p95": 0.13722771406173706 }, { "label": "current_group_partial", "mean": 0.18827059864997864, "median": 0.1826695203781128, "p05": 0.13249869644641876, "p95": 0.2633744776248932 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6546597480773926, "group": 5, "latest_source_mass": 0.35775136947631836, "layer": 18, "max_source_mass": 0.35775136947631836, "mixer_index": 34, "normalized_entropy": 0.9234831887286026, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.17577320337295532, "median": 0.17441116273403168, "p05": 0.12769785523414612, "p95": 0.226454958319664 }, { "label": "completed_group_1", "mean": 0.14207448065280914, "median": 0.14135520160198212, "p05": 0.12424421310424805, "p95": 0.16169092059135437 }, { "label": "completed_group_2", "mean": 0.08111411333084106, "median": 0.08080219477415085, "p05": 0.071528859436512, "p95": 0.09159335494041443 }, { "label": "completed_group_3", "mean": 0.0931762307882309, "median": 0.09094840288162231, "p05": 0.0776137039065361, "p95": 0.11699455231428146 }, { "label": "completed_group_4", "mean": 0.15011057257652283, "median": 0.14907215535640717, "p05": 0.1309027075767517, "p95": 0.17212367057800293 }, { "label": "current_group_partial", "mean": 0.35775136947631836, "median": 0.35810214281082153, "p05": 0.3160427212715149, "p95": 0.4006560444831848 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.430190086364746, "group": 5, "latest_source_mass": 0.5245762467384338, "layer": 18, "max_source_mass": 0.5245762467384338, "mixer_index": 35, "normalized_entropy": 0.7982042851884109, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1354856789112091, "median": 0.1260826736688614, "p05": 0.08359937369823456, "p95": 0.2213967740535736 }, { "label": "completed_group_1", "mean": 0.09479983896017075, "median": 0.09236767888069153, "p05": 0.0743592232465744, "p95": 0.12202530354261398 }, { "label": "completed_group_2", "mean": 0.07003438472747803, "median": 0.06948544830083847, "p05": 0.05187927931547165, "p95": 0.09030532836914062 }, { "label": "completed_group_3", "mean": 0.083448626101017, "median": 0.080234095454216, "p05": 0.06169745698571205, "p95": 0.11337985098361969 }, { "label": "completed_group_4", "mean": 0.09165525436401367, "median": 0.09136426448822021, "p05": 0.07385377585887909, "p95": 0.11038129031658173 }, { "label": "current_group_partial", "mean": 0.5245762467384338, "median": 0.5328655242919922, "p05": 0.42046138644218445, "p95": 0.60233473777771 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7008471488952637, "group": 5, "latest_source_mass": 0.31297987699508667, "layer": 19, "max_source_mass": 0.31297987699508667, "mixer_index": 36, "normalized_entropy": 0.9492608679378381, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18653985857963562, "median": 0.18515554070472717, "p05": 0.14839154481887817, "p95": 0.2285771518945694 }, { "label": "completed_group_1", "mean": 0.149653822183609, "median": 0.14952652156352997, "p05": 0.1333337277173996, "p95": 0.16640914976596832 }, { "label": "completed_group_2", "mean": 0.09563065320253372, "median": 0.0953449159860611, "p05": 0.08531444519758224, "p95": 0.10701718926429749 }, { "label": "completed_group_3", "mean": 0.09960515052080154, "median": 0.09781023114919662, "p05": 0.08538899570703506, "p95": 0.12114660441875458 }, { "label": "completed_group_4", "mean": 0.15559062361717224, "median": 0.15579181909561157, "p05": 0.14112713932991028, "p95": 0.16977989673614502 }, { "label": "current_group_partial", "mean": 0.31297987699508667, "median": 0.3142726719379425, "p05": 0.2775314450263977, "p95": 0.3449920415878296 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.514115333557129, "group": 5, "latest_source_mass": 0.4571846127510071, "layer": 19, "max_source_mass": 0.4571846127510071, "mixer_index": 37, "normalized_entropy": 0.84504385748242, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14724519848823547, "median": 0.13547402620315552, "p05": 0.08005326241254807, "p95": 0.2764008939266205 }, { "label": "completed_group_1", "mean": 0.13274207711219788, "median": 0.13091760873794556, "p05": 0.1022145226597786, "p95": 0.1677413284778595 }, { "label": "completed_group_2", "mean": 0.096603162586689, "median": 0.09685055911540985, "p05": 0.07085022330284119, "p95": 0.12084522098302841 }, { "label": "completed_group_3", "mean": 0.1134733334183693, "median": 0.11085811257362366, "p05": 0.08714926987886429, "p95": 0.14451190829277039 }, { "label": "completed_group_4", "mean": 0.0527515783905983, "median": 0.05150189250707626, "p05": 0.03931276127696037, "p95": 0.07069683074951172 }, { "label": "current_group_partial", "mean": 0.4571846127510071, "median": 0.4572353661060333, "p05": 0.3533322513103485, "p95": 0.5581004023551941 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7213332653045654, "group": 5, "latest_source_mass": 0.2855108678340912, "layer": 20, "max_source_mass": 0.2855108678340912, "mixer_index": 38, "normalized_entropy": 0.9606943872026353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19527462124824524, "median": 0.19608022272586823, "p05": 0.14261451363563538, "p95": 0.2492632269859314 }, { "label": "completed_group_1", "mean": 0.16114139556884766, "median": 0.16161862015724182, "p05": 0.14146453142166138, "p95": 0.18065661191940308 }, { "label": "completed_group_2", "mean": 0.10406571626663208, "median": 0.10396719723939896, "p05": 0.09171589463949203, "p95": 0.11652514338493347 }, { "label": "completed_group_3", "mean": 0.10423287749290466, "median": 0.10206204652786255, "p05": 0.08988140523433685, "p95": 0.12570036947727203 }, { "label": "completed_group_4", "mean": 0.14977452158927917, "median": 0.14978185296058655, "p05": 0.13461759686470032, "p95": 0.1649411916732788 }, { "label": "current_group_partial", "mean": 0.2855108678340912, "median": 0.28573572635650635, "p05": 0.24868330359458923, "p95": 0.32105109095573425 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.503009557723999, "group": 5, "latest_source_mass": 0.44554412364959717, "layer": 20, "max_source_mass": 0.44554412364959717, "mixer_index": 39, "normalized_entropy": 0.8388456059738542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1677132397890091, "median": 0.14348560571670532, "p05": 0.08334699273109436, "p95": 0.36093956232070923 }, { "label": "completed_group_1", "mean": 0.12867671251296997, "median": 0.12582841515541077, "p05": 0.09450241923332214, "p95": 0.17028142511844635 }, { "label": "completed_group_2", "mean": 0.09363879263401031, "median": 0.0947161316871643, "p05": 0.06503768265247345, "p95": 0.11920955032110214 }, { "label": "completed_group_3", "mean": 0.12237932533025742, "median": 0.11932459473609924, "p05": 0.09139132499694824, "p95": 0.15721726417541504 }, { "label": "completed_group_4", "mean": 0.04204780235886574, "median": 0.0400366373360157, "p05": 0.027764063328504562, "p95": 0.06483393162488937 }, { "label": "current_group_partial", "mean": 0.44554412364959717, "median": 0.44789671897888184, "p05": 0.2981249690055847, "p95": 0.563161313533783 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7531253099441528, "group": 6, "latest_source_mass": 0.22089648246765137, "layer": 21, "max_source_mass": 0.22089648246765137, "mixer_index": 40, "normalized_entropy": 0.9784378651557807, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21246549487113953, "median": 0.2120085060596466, "p05": 0.16239126026630402, "p95": 0.2642289698123932 }, { "label": "completed_group_1", "mean": 0.17559616267681122, "median": 0.17583107948303223, "p05": 0.15805834531784058, "p95": 0.19371670484542847 }, { "label": "completed_group_2", "mean": 0.11484665423631668, "median": 0.11523692309856415, "p05": 0.10031608492136002, "p95": 0.12835028767585754 }, { "label": "completed_group_3", "mean": 0.11058571934700012, "median": 0.10834631323814392, "p05": 0.09625516831874847, "p95": 0.13235723972320557 }, { "label": "completed_group_4", "mean": 0.1656094789505005, "median": 0.1659335494041443, "p05": 0.14772318303585052, "p95": 0.18170902132987976 }, { "label": "completed_group_5", "mean": 0.22089648246765137, "median": 0.22141876816749573, "p05": 0.1899021565914154, "p95": 0.24976326525211334 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3981997966766357, "group": 6, "latest_source_mass": 0.5701477527618408, "layer": 21, "max_source_mass": 0.5701477527618408, "mixer_index": 41, "normalized_entropy": 0.7185325578138456, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08501994609832764, "median": 0.07170915603637695, "p05": 0.0405949205160141, "p95": 0.19168326258659363 }, { "label": "completed_group_1", "mean": 0.06276540458202362, "median": 0.061594001948833466, "p05": 0.04767107218503952, "p95": 0.08238202333450317 }, { "label": "completed_group_2", "mean": 0.0543542318046093, "median": 0.054001111537218094, "p05": 0.039771370589733124, "p95": 0.07025467604398727 }, { "label": "completed_group_3", "mean": 0.06455785036087036, "median": 0.06315328925848007, "p05": 0.046149056404829025, "p95": 0.08615517616271973 }, { "label": "completed_group_4", "mean": 0.03336101397871971, "median": 0.03271401301026344, "p05": 0.025995943695306778, "p95": 0.04298749938607216 }, { "label": "completed_group_5", "mean": 0.12979380786418915, "median": 0.1275225281715393, "p05": 0.09634383022785187, "p95": 0.16848969459533691 }, { "label": "current_group_partial", "mean": 0.5701477527618408, "median": 0.5752488374710083, "p05": 0.4706467390060425, "p95": 0.6460088491439819 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.6064693927764893, "group": 6, "latest_source_mass": 0.4729730188846588, "layer": 22, "max_source_mass": 0.4729730188846588, "mixer_index": 42, "normalized_entropy": 0.8255619580155779, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12975823879241943, "median": 0.13364392518997192, "p05": 0.07449157536029816, "p95": 0.17710214853286743 }, { "label": "completed_group_1", "mean": 0.09198253601789474, "median": 0.09123716503381729, "p05": 0.07526302337646484, "p95": 0.11214317381381989 }, { "label": "completed_group_2", "mean": 0.061632055789232254, "median": 0.06144782900810242, "p05": 0.05014941096305847, "p95": 0.07370813935995102 }, { "label": "completed_group_3", "mean": 0.06021754443645477, "median": 0.05830872803926468, "p05": 0.048907674849033356, "p95": 0.07788939028978348 }, { "label": "completed_group_4", "mean": 0.078399658203125, "median": 0.07768932729959488, "p05": 0.06658457219600677, "p95": 0.09283287078142166 }, { "label": "completed_group_5", "mean": 0.10503692924976349, "median": 0.1041785180568695, "p05": 0.08639058470726013, "p95": 0.12733891606330872 }, { "label": "current_group_partial", "mean": 0.4729730188846588, "median": 0.46905672550201416, "p05": 0.40225255489349365, "p95": 0.5535125136375427 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3297276496887207, "group": 6, "latest_source_mass": 0.6041883230209351, "layer": 22, "max_source_mass": 0.6041883230209351, "mixer_index": 43, "normalized_entropy": 0.6833448349782582, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09028719365596771, "median": 0.0814780443906784, "p05": 0.04628303647041321, "p95": 0.17968304455280304 }, { "label": "completed_group_1", "mean": 0.057085148990154266, "median": 0.05594421550631523, "p05": 0.040768980979919434, "p95": 0.07630246877670288 }, { "label": "completed_group_2", "mean": 0.04790058732032776, "median": 0.04758875072002411, "p05": 0.033580001443624496, "p95": 0.06301991641521454 }, { "label": "completed_group_3", "mean": 0.05685827136039734, "median": 0.0552905797958374, "p05": 0.037766557186841965, "p95": 0.07916736602783203 }, { "label": "completed_group_4", "mean": 0.031231366097927094, "median": 0.030573323369026184, "p05": 0.023405978456139565, "p95": 0.041360143572092056 }, { "label": "completed_group_5", "mean": 0.11244906485080719, "median": 0.11069123446941376, "p05": 0.07942862808704376, "p95": 0.1529570370912552 }, { "label": "current_group_partial", "mean": 0.6041883230209351, "median": 0.6044443845748901, "p05": 0.495843768119812, "p95": 0.7062365412712097 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.6772459745407104, "group": 6, "latest_source_mass": 0.4253680109977722, "layer": 23, "max_source_mass": 0.4253680109977722, "mixer_index": 44, "normalized_entropy": 0.8619339260628082, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1455874890089035, "median": 0.15217238664627075, "p05": 0.08379869163036346, "p95": 0.19571389257907867 }, { "label": "completed_group_1", "mean": 0.10527922213077545, "median": 0.10456782579421997, "p05": 0.08953425288200378, "p95": 0.12367816269397736 }, { "label": "completed_group_2", "mean": 0.06669704616069794, "median": 0.06652139127254486, "p05": 0.05353657156229019, "p95": 0.0801846832036972 }, { "label": "completed_group_3", "mean": 0.06069611385464668, "median": 0.058348577469587326, "p05": 0.048359066247940063, "p95": 0.08090560138225555 }, { "label": "completed_group_4", "mean": 0.09951996803283691, "median": 0.09863626956939697, "p05": 0.08421580493450165, "p95": 0.11803246289491653 }, { "label": "completed_group_5", "mean": 0.09685218334197998, "median": 0.09553758800029755, "p05": 0.07873044908046722, "p95": 0.11939825117588043 }, { "label": "current_group_partial", "mean": 0.4253680109977722, "median": 0.4225534200668335, "p05": 0.3542138934135437, "p95": 0.5074539184570312 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3742375373840332, "group": 6, "latest_source_mass": 0.5837908983230591, "layer": 23, "max_source_mass": 0.5837908983230591, "mixer_index": 45, "normalized_entropy": 0.706218392483943, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09774194657802582, "median": 0.0849265456199646, "p05": 0.05457359179854393, "p95": 0.19752079248428345 }, { "label": "completed_group_1", "mean": 0.06367622315883636, "median": 0.062340207397937775, "p05": 0.047256506979465485, "p95": 0.0836888924241066 }, { "label": "completed_group_2", "mean": 0.05300067365169525, "median": 0.052215054631233215, "p05": 0.03693629428744316, "p95": 0.07218416780233383 }, { "label": "completed_group_3", "mean": 0.05981069058179855, "median": 0.05713159963488579, "p05": 0.039428286254405975, "p95": 0.08748359233140945 }, { "label": "completed_group_4", "mean": 0.03241024166345596, "median": 0.03175225481390953, "p05": 0.024235714226961136, "p95": 0.042685288935899734 }, { "label": "completed_group_5", "mean": 0.10956928879022598, "median": 0.10649192333221436, "p05": 0.07568290829658508, "p95": 0.15524834394454956 }, { "label": "current_group_partial", "mean": 0.5837908983230591, "median": 0.5887123346328735, "p05": 0.4634547531604767, "p95": 0.6879903078079224 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.672610878944397, "group": 6, "latest_source_mass": 0.42961159348487854, "layer": 24, "max_source_mass": 0.42961159348487854, "mixer_index": 46, "normalized_entropy": 0.8595519581191374, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14811497926712036, "median": 0.15288400650024414, "p05": 0.08557615429162979, "p95": 0.20698775351047516 }, { "label": "completed_group_1", "mean": 0.10026681423187256, "median": 0.09941448271274567, "p05": 0.08483528345823288, "p95": 0.11770403385162354 }, { "label": "completed_group_2", "mean": 0.0711982250213623, "median": 0.07080988585948944, "p05": 0.05726742744445801, "p95": 0.08595790714025497 }, { "label": "completed_group_3", "mean": 0.06714795529842377, "median": 0.06442591547966003, "p05": 0.052491698414087296, "p95": 0.0918450579047203 }, { "label": "completed_group_4", "mean": 0.08556140959262848, "median": 0.0845319926738739, "p05": 0.0714617371559143, "p95": 0.1041039377450943 }, { "label": "completed_group_5", "mean": 0.09809904545545578, "median": 0.09676970541477203, "p05": 0.07911732792854309, "p95": 0.12164569646120071 }, { "label": "current_group_partial", "mean": 0.42961159348487854, "median": 0.42717063426971436, "p05": 0.3544589579105377, "p95": 0.5187017321586609 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4325350522994995, "group": 6, "latest_source_mass": 0.5540482997894287, "layer": 24, "max_source_mass": 0.5540482997894287, "mixer_index": 47, "normalized_entropy": 0.736177388763277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11123399436473846, "median": 0.09619481861591339, "p05": 0.04875103384256363, "p95": 0.24804341793060303 }, { "label": "completed_group_1", "mean": 0.07182647287845612, "median": 0.07066938281059265, "p05": 0.054608479142189026, "p95": 0.09248427301645279 }, { "label": "completed_group_2", "mean": 0.05620439723134041, "median": 0.055612146854400635, "p05": 0.039178267121315, "p95": 0.07496444135904312 }, { "label": "completed_group_3", "mean": 0.06410455703735352, "median": 0.06079697608947754, "p05": 0.04396975412964821, "p95": 0.09297186136245728 }, { "label": "completed_group_4", "mean": 0.03741488978266716, "median": 0.03642338514328003, "p05": 0.027955297380685806, "p95": 0.05010963976383209 }, { "label": "completed_group_5", "mean": 0.10516737401485443, "median": 0.10250729322433472, "p05": 0.07237984985113144, "p95": 0.14790719747543335 }, { "label": "current_group_partial", "mean": 0.5540482997894287, "median": 0.5591273903846741, "p05": 0.42828047275543213, "p95": 0.6615095138549805 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7746202945709229, "group": 7, "latest_source_mass": 0.3583717346191406, "layer": 25, "max_source_mass": 0.3583717346191406, "mixer_index": 48, "normalized_entropy": 0.911974427715716, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15383490920066833, "median": 0.1525440216064453, "p05": 0.09242676198482513, "p95": 0.2355288565158844 }, { "label": "completed_group_1", "mean": 0.11435382068157196, "median": 0.11365524679422379, "p05": 0.09744305908679962, "p95": 0.13308049738407135 }, { "label": "completed_group_2", "mean": 0.0908353179693222, "median": 0.09055374562740326, "p05": 0.07310554385185242, "p95": 0.109998419880867 }, { "label": "completed_group_3", "mean": 0.08329387754201889, "median": 0.08085054904222488, "p05": 0.06748563796281815, "p95": 0.10712415724992752 }, { "label": "completed_group_4", "mean": 0.08837151527404785, "median": 0.0870782658457756, "p05": 0.07312256097793579, "p95": 0.10766587406396866 }, { "label": "completed_group_5", "mean": 0.11093883961439133, "median": 0.11001912504434586, "p05": 0.08911329507827759, "p95": 0.1362026184797287 }, { "label": "completed_group_6", "mean": 0.3583717346191406, "median": 0.35752072930336, "p05": 0.2904500365257263, "p95": 0.4329596757888794 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5557539463043213, "group": 7, "latest_source_mass": 0.27552926540374756, "layer": 25, "max_source_mass": 0.40426212549209595, "mixer_index": 49, "normalized_entropy": 0.7481595010588931, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09889799356460571, "median": 0.09010560810565948, "p05": 0.052236538380384445, "p95": 0.17090652883052826 }, { "label": "completed_group_1", "mean": 0.05227670818567276, "median": 0.05044294521212578, "p05": 0.03345092013478279, "p95": 0.0764506608247757 }, { "label": "completed_group_2", "mean": 0.03886467218399048, "median": 0.03779824823141098, "p05": 0.024015849456191063, "p95": 0.05721688270568848 }, { "label": "completed_group_3", "mean": 0.039664678275585175, "median": 0.03654132038354874, "p05": 0.02151499129831791, "p95": 0.06773733347654343 }, { "label": "completed_group_4", "mean": 0.02741251140832901, "median": 0.026939166709780693, "p05": 0.019440561532974243, "p95": 0.03718416765332222 }, { "label": "completed_group_5", "mean": 0.06309205293655396, "median": 0.05998644977807999, "p05": 0.03781827166676521, "p95": 0.09845902025699615 }, { "label": "completed_group_6", "mean": 0.40426212549209595, "median": 0.4023381471633911, "p05": 0.21087685227394104, "p95": 0.595990002155304 }, { "label": "current_group_partial", "mean": 0.27552926540374756, "median": 0.27764469385147095, "p05": 0.05720072239637375, "p95": 0.5175203680992126 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7942585945129395, "group": 7, "latest_source_mass": 0.322842001914978, "layer": 26, "max_source_mass": 0.322842001914978, "mixer_index": 50, "normalized_entropy": 0.8628559921254064, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10892649739980698, "median": 0.11189229786396027, "p05": 0.03843415528535843, "p95": 0.1755073070526123 }, { "label": "completed_group_1", "mean": 0.08669496327638626, "median": 0.08709807693958282, "p05": 0.051237381994724274, "p95": 0.12081066519021988 }, { "label": "completed_group_2", "mean": 0.06022954732179642, "median": 0.059791453182697296, "p05": 0.04038726165890694, "p95": 0.08276855945587158 }, { "label": "completed_group_3", "mean": 0.05406671017408371, "median": 0.051769405603408813, "p05": 0.03718401864171028, "p95": 0.07747116684913635 }, { "label": "completed_group_4", "mean": 0.05429235100746155, "median": 0.05316123366355896, "p05": 0.040919527411460876, "p95": 0.07187244296073914 }, { "label": "completed_group_5", "mean": 0.07220236957073212, "median": 0.07145723700523376, "p05": 0.052494119852781296, "p95": 0.09450244903564453 }, { "label": "completed_group_6", "mean": 0.24074554443359375, "median": 0.23948168754577637, "p05": 0.15157052874565125, "p95": 0.33065202832221985 }, { "label": "current_group_partial", "mean": 0.322842001914978, "median": 0.31601065397262573, "p05": 0.15581756830215454, "p95": 0.5172116756439209 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6406831741333008, "group": 7, "latest_source_mass": 0.3283761143684387, "layer": 26, "max_source_mass": 0.3283761143684387, "mixer_index": 51, "normalized_entropy": 0.7890018263306923, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08106359094381332, "median": 0.06559887528419495, "p05": 0.031218135729432106, "p95": 0.21695302426815033 }, { "label": "completed_group_1", "mean": 0.06122073531150818, "median": 0.059359729290008545, "p05": 0.03888052701950073, "p95": 0.087535060942173 }, { "label": "completed_group_2", "mean": 0.04382575303316116, "median": 0.04286644607782364, "p05": 0.029065817594528198, "p95": 0.061862632632255554 }, { "label": "completed_group_3", "mean": 0.0495278537273407, "median": 0.046749986708164215, "p05": 0.02850426360964775, "p95": 0.07717642933130264 }, { "label": "completed_group_4", "mean": 0.031025834381580353, "median": 0.03045039065182209, "p05": 0.0226125568151474, "p95": 0.04110690951347351 }, { "label": "completed_group_5", "mean": 0.08371539413928986, "median": 0.08070263266563416, "p05": 0.04838854819536209, "p95": 0.1307208389043808 }, { "label": "completed_group_6", "mean": 0.3212447166442871, "median": 0.3227238059043884, "p05": 0.1668301224708557, "p95": 0.46654704213142395 }, { "label": "current_group_partial", "mean": 0.3283761143684387, "median": 0.31801193952560425, "p05": 0.1346617341041565, "p95": 0.5606002807617188 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8771746158599854, "group": 7, "latest_source_mass": 0.2884133756160736, "layer": 27, "max_source_mass": 0.2884133756160736, "mixer_index": 52, "normalized_entropy": 0.9027301697279486, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11823929846286774, "median": 0.1204136461019516, "p05": 0.06847050786018372, "p95": 0.16903573274612427 }, { "label": "completed_group_1", "mean": 0.09187045693397522, "median": 0.0907389223575592, "p05": 0.06882993876934052, "p95": 0.11873672157526016 }, { "label": "completed_group_2", "mean": 0.06966834515333176, "median": 0.06954923272132874, "p05": 0.052450068295001984, "p95": 0.08739262074232101 }, { "label": "completed_group_3", "mean": 0.05918116122484207, "median": 0.05728268623352051, "p05": 0.043897710740566254, "p95": 0.08088754862546921 }, { "label": "completed_group_4", "mean": 0.06588791310787201, "median": 0.06536087393760681, "p05": 0.05338210612535477, "p95": 0.08069764077663422 }, { "label": "completed_group_5", "mean": 0.07802121341228485, "median": 0.0768343061208725, "p05": 0.06256727874279022, "p95": 0.09689624607563019 }, { "label": "completed_group_6", "mean": 0.22871825098991394, "median": 0.22734758257865906, "p05": 0.16477933526039124, "p95": 0.2942171096801758 }, { "label": "current_group_partial", "mean": 0.2884133756160736, "median": 0.2829434871673584, "p05": 0.17691120505332947, "p95": 0.41698765754699707 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.612527847290039, "group": 7, "latest_source_mass": 0.35910409688949585, "layer": 27, "max_source_mass": 0.35910409688949585, "mixer_index": 53, "normalized_entropy": 0.775461976193565, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09160975366830826, "median": 0.08172477036714554, "p05": 0.04350627586245537, "p95": 0.18076449632644653 }, { "label": "completed_group_1", "mean": 0.05898914858698845, "median": 0.05659175664186478, "p05": 0.03745892643928528, "p95": 0.08595352619886398 }, { "label": "completed_group_2", "mean": 0.039636120200157166, "median": 0.037070050835609436, "p05": 0.022959833964705467, "p95": 0.06498780846595764 }, { "label": "completed_group_3", "mean": 0.04581688344478607, "median": 0.0417281799018383, "p05": 0.02348574623465538, "p95": 0.07863478362560272 }, { "label": "completed_group_4", "mean": 0.032106831669807434, "median": 0.03162588179111481, "p05": 0.022784041240811348, "p95": 0.04321740195155144 }, { "label": "completed_group_5", "mean": 0.06671778857707977, "median": 0.06372548639774323, "p05": 0.038862183690071106, "p95": 0.1047079786658287 }, { "label": "completed_group_6", "mean": 0.3060194253921509, "median": 0.30642110109329224, "p05": 0.14355266094207764, "p95": 0.46712520718574524 }, { "label": "current_group_partial", "mean": 0.35910409688949585, "median": 0.3470078706741333, "p05": 0.15172286331653595, "p95": 0.5934250950813293 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8253357410430908, "group": 7, "latest_source_mass": 0.2919178009033203, "layer": 28, "max_source_mass": 0.2919178009033203, "mixer_index": 54, "normalized_entropy": 0.8778009405200828, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10355445742607117, "median": 0.1021876335144043, "p05": 0.0484653078019619, "p95": 0.16425444185733795 }, { "label": "completed_group_1", "mean": 0.08380813896656036, "median": 0.08186767250299454, "p05": 0.05864008516073227, "p95": 0.11570068448781967 }, { "label": "completed_group_2", "mean": 0.06609268486499786, "median": 0.0652528628706932, "p05": 0.04682006686925888, "p95": 0.08782888948917389 }, { "label": "completed_group_3", "mean": 0.07030214369297028, "median": 0.06750299036502838, "p05": 0.04772508144378662, "p95": 0.10195659846067429 }, { "label": "completed_group_4", "mean": 0.04850628226995468, "median": 0.047426991164684296, "p05": 0.03554021194577217, "p95": 0.0651378333568573 }, { "label": "completed_group_5", "mean": 0.0775601714849472, "median": 0.07640299201011658, "p05": 0.05519672483205795, "p95": 0.10486777871847153 }, { "label": "completed_group_6", "mean": 0.25825831294059753, "median": 0.2556683421134949, "p05": 0.16664613783359528, "p95": 0.35751956701278687 }, { "label": "current_group_partial", "mean": 0.2919178009033203, "median": 0.2851294279098511, "p05": 0.14807775616645813, "p95": 0.4601541757583618 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.714170217514038, "group": 7, "latest_source_mass": 0.25887250900268555, "layer": 28, "max_source_mass": 0.3351396918296814, "mixer_index": 55, "normalized_entropy": 0.8243416240156862, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09696917235851288, "median": 0.07196179032325745, "p05": 0.02744467183947563, "p95": 0.2601720094680786 }, { "label": "completed_group_1", "mean": 0.07828262448310852, "median": 0.07698164135217667, "p05": 0.05464315786957741, "p95": 0.10479431599378586 }, { "label": "completed_group_2", "mean": 0.05506835877895355, "median": 0.054356254637241364, "p05": 0.03805546090006828, "p95": 0.07392264902591705 }, { "label": "completed_group_3", "mean": 0.0616651214659214, "median": 0.058430496603250504, "p05": 0.0378233827650547, "p95": 0.09199885278940201 }, { "label": "completed_group_4", "mean": 0.03598042577505112, "median": 0.03550964966416359, "p05": 0.025762276723980904, "p95": 0.04832924157381058 }, { "label": "completed_group_5", "mean": 0.07802211493253708, "median": 0.07588380575180054, "p05": 0.05081838741898537, "p95": 0.1116337925195694 }, { "label": "completed_group_6", "mean": 0.3351396918296814, "median": 0.34016722440719604, "p05": 0.1802617907524109, "p95": 0.47657349705696106 }, { "label": "current_group_partial", "mean": 0.25887250900268555, "median": 0.2348807156085968, "p05": 0.08903636038303375, "p95": 0.5087121725082397 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.875918984413147, "group": 8, "latest_source_mass": 0.22883272171020508, "layer": 29, "max_source_mass": 0.2776731252670288, "mixer_index": 56, "normalized_entropy": 0.9021263386407693, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11156761646270752, "median": 0.10779387503862381, "p05": 0.05963520333170891, "p95": 0.17809909582138062 }, { "label": "completed_group_1", "mean": 0.0880265012383461, "median": 0.08795224130153656, "p05": 0.05159609019756317, "p95": 0.12478911876678467 }, { "label": "completed_group_2", "mean": 0.07785218209028244, "median": 0.0769609659910202, "p05": 0.052540622651576996, "p95": 0.1063356101512909 }, { "label": "completed_group_3", "mean": 0.07930341362953186, "median": 0.076581209897995, "p05": 0.051584642380476, "p95": 0.11473879218101501 }, { "label": "completed_group_4", "mean": 0.04990004748106003, "median": 0.048692211508750916, "p05": 0.03663463890552521, "p95": 0.06705896556377411 }, { "label": "completed_group_5", "mean": 0.08684436976909637, "median": 0.08598007261753082, "p05": 0.06167534738779068, "p95": 0.11696360260248184 }, { "label": "completed_group_6", "mean": 0.2776731252670288, "median": 0.27646756172180176, "p05": 0.18715886771678925, "p95": 0.37483638525009155 }, { "label": "completed_group_7", "mean": 0.22883272171020508, "median": 0.21553385257720947, "p05": 0.11123093962669373, "p95": 0.38216158747673035 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.731968641281128, "group": 8, "latest_source_mass": 0.3927639126777649, "layer": 29, "max_source_mass": 0.3927639126777649, "mixer_index": 57, "normalized_entropy": 0.788252898290834, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.06617744266986847, "median": 0.05556660145521164, "p05": 0.026227911934256554, "p95": 0.14206086099147797 }, { "label": "completed_group_1", "mean": 0.05556279420852661, "median": 0.052200887352228165, "p05": 0.03175564482808113, "p95": 0.09087387472391129 }, { "label": "completed_group_2", "mean": 0.043194323778152466, "median": 0.04139059782028198, "p05": 0.026660459116101265, "p95": 0.06614840030670166 }, { "label": "completed_group_3", "mean": 0.045391663908958435, "median": 0.04136885702610016, "p05": 0.02558080479502678, "p95": 0.0755799189209938 }, { "label": "completed_group_4", "mean": 0.026374589651823044, "median": 0.025806495919823647, "p05": 0.01769660972058773, "p95": 0.03712644428014755 }, { "label": "completed_group_5", "mean": 0.054588086903095245, "median": 0.052078139036893845, "p05": 0.03126347064971924, "p95": 0.08616672456264496 }, { "label": "completed_group_6", "mean": 0.21148523688316345, "median": 0.20441749691963196, "p05": 0.11578395217657089, "p95": 0.3317175507545471 }, { "label": "completed_group_7", "mean": 0.10446193814277649, "median": 0.08533146977424622, "p05": 0.03682969510555267, "p95": 0.23046134412288666 }, { "label": "current_group_partial", "mean": 0.3927639126777649, "median": 0.398543119430542, "p05": 0.17646673321723938, "p95": 0.5867685079574585 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9649744033813477, "group": 8, "latest_source_mass": 0.18465125560760498, "layer": 30, "max_source_mass": 0.27242881059646606, "mixer_index": 58, "normalized_entropy": 0.8942983906376845, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07813189923763275, "median": 0.05827457085251808, "p05": 0.030488120391964912, "p95": 0.23055212199687958 }, { "label": "completed_group_1", "mean": 0.08751582354307175, "median": 0.08534294366836548, "p05": 0.05223194509744644, "p95": 0.12876519560813904 }, { "label": "completed_group_2", "mean": 0.05837182328104973, "median": 0.05781412869691849, "p05": 0.042913295328617096, "p95": 0.07544869184494019 }, { "label": "completed_group_3", "mean": 0.07608449459075928, "median": 0.0739547461271286, "p05": 0.05176331102848053, "p95": 0.10407407581806183 }, { "label": "completed_group_4", "mean": 0.031109986826777458, "median": 0.029160991311073303, "p05": 0.02102847397327423, "p95": 0.04819672927260399 }, { "label": "completed_group_5", "mean": 0.09630586206912994, "median": 0.09455011785030365, "p05": 0.06405375152826309, "p95": 0.1344345211982727 }, { "label": "completed_group_6", "mean": 0.27242881059646606, "median": 0.2743605375289917, "p05": 0.1689583659172058, "p95": 0.373348206281662 }, { "label": "completed_group_7", "mean": 0.1154000535607338, "median": 0.1035175770521164, "p05": 0.04644457623362541, "p95": 0.2236812710762024 }, { "label": "current_group_partial", "mean": 0.18465125560760498, "median": 0.17620417475700378, "p05": 0.09633602201938629, "p95": 0.30151551961898804 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.8779311180114746, "group": 8, "latest_source_mass": 0.24353644251823425, "layer": 30, "max_source_mass": 0.26414474844932556, "mixer_index": 59, "normalized_entropy": 0.8546832842586183, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07922039180994034, "median": 0.06023593619465828, "p05": 0.03731776028871536, "p95": 0.21911568939685822 }, { "label": "completed_group_1", "mean": 0.064256951212883, "median": 0.06290853023529053, "p05": 0.041900213807821274, "p95": 0.08994727581739426 }, { "label": "completed_group_2", "mean": 0.04481552541255951, "median": 0.04388217628002167, "p05": 0.029372509568929672, "p95": 0.06349608302116394 }, { "label": "completed_group_3", "mean": 0.05368471518158913, "median": 0.05044667050242424, "p05": 0.03311370313167572, "p95": 0.08277972787618637 }, { "label": "completed_group_4", "mean": 0.03291679173707962, "median": 0.03245514631271362, "p05": 0.023155367001891136, "p95": 0.044260501861572266 }, { "label": "completed_group_5", "mean": 0.07845190167427063, "median": 0.07614757120609283, "p05": 0.04700833186507225, "p95": 0.11541540920734406 }, { "label": "completed_group_6", "mean": 0.26414474844932556, "median": 0.2620946168899536, "p05": 0.13117244839668274, "p95": 0.4018894135951996 }, { "label": "completed_group_7", "mean": 0.13897252082824707, "median": 0.11859199404716492, "p05": 0.050003524869680405, "p95": 0.2953890562057495 }, { "label": "current_group_partial", "mean": 0.24353644251823425, "median": 0.23090004920959473, "p05": 0.0989413633942604, "p95": 0.43275272846221924 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9959204196929932, "group": 8, "latest_source_mass": 0.177350252866745, "layer": 31, "max_source_mass": 0.23861892521381378, "mixer_index": 60, "normalized_entropy": 0.9083825296150313, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10073552280664444, "median": 0.10023926943540573, "p05": 0.015173540450632572, "p95": 0.2112843543291092 }, { "label": "completed_group_1", "mean": 0.0823616161942482, "median": 0.07981257140636444, "p05": 0.05091645568609238, "p95": 0.12525078654289246 }, { "label": "completed_group_2", "mean": 0.0686727911233902, "median": 0.06696957349777222, "p05": 0.045773059129714966, "p95": 0.09787814319133759 }, { "label": "completed_group_3", "mean": 0.0658436119556427, "median": 0.06227286905050278, "p05": 0.04276903346180916, "p95": 0.09779240190982819 }, { "label": "completed_group_4", "mean": 0.048376429826021194, "median": 0.04572375863790512, "p05": 0.033916816115379333, "p95": 0.07051397114992142 }, { "label": "completed_group_5", "mean": 0.07448098063468933, "median": 0.07275891304016113, "p05": 0.051099833101034164, "p95": 0.10395565629005432 }, { "label": "completed_group_6", "mean": 0.23861892521381378, "median": 0.238796204328537, "p05": 0.14983810484409332, "p95": 0.3338800072669983 }, { "label": "completed_group_7", "mean": 0.14355985820293427, "median": 0.12910249829292297, "p05": 0.051254600286483765, "p95": 0.2900521159172058 }, { "label": "current_group_partial", "mean": 0.177350252866745, "median": 0.17761719226837158, "p05": 0.060382649302482605, "p95": 0.30869320034980774 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9298079013824463, "group": 8, "latest_source_mass": 0.2151678204536438, "layer": 31, "max_source_mass": 0.2151678204536438, "mixer_index": 61, "normalized_entropy": 0.8782934258463589, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09570661187171936, "median": 0.09196878969669342, "p05": 0.026315581053495407, "p95": 0.18887758255004883 }, { "label": "completed_group_1", "mean": 0.07176163792610168, "median": 0.06893502175807953, "p05": 0.04786262661218643, "p95": 0.10555325448513031 }, { "label": "completed_group_2", "mean": 0.05499213933944702, "median": 0.05241403728723526, "p05": 0.03552260994911194, "p95": 0.08411884307861328 }, { "label": "completed_group_3", "mean": 0.05746772885322571, "median": 0.05466804280877113, "p05": 0.03474465757608414, "p95": 0.08745065331459045 }, { "label": "completed_group_4", "mean": 0.03367581218481064, "median": 0.03265039622783661, "p05": 0.023353515192866325, "p95": 0.047148022800683975 }, { "label": "completed_group_5", "mean": 0.06890923529863358, "median": 0.0663483589887619, "p05": 0.03880944103002548, "p95": 0.1076190173625946 }, { "label": "completed_group_6", "mean": 0.21427617967128754, "median": 0.2086736112833023, "p05": 0.11234137415885925, "p95": 0.3368087410926819 }, { "label": "completed_group_7", "mean": 0.18804283440113068, "median": 0.1678360402584076, "p05": 0.06922265142202377, "p95": 0.37698543071746826 }, { "label": "current_group_partial", "mean": 0.2151678204536438, "median": 0.21195992827415466, "p05": 0.07000727951526642, "p95": 0.3764724135398865 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.8635971546173096, "group": 8, "latest_source_mass": 0.10272546857595444, "layer": 32, "max_source_mass": 0.3632875978946686, "mixer_index": 62, "normalized_entropy": 0.8481596163814172, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.07685244083404541, "median": 0.050476111471652985, "p05": 0.021709207445383072, "p95": 0.25511038303375244 }, { "label": "completed_group_1", "mean": 0.07133443653583527, "median": 0.06903944909572601, "p05": 0.041264068335294724, "p95": 0.1117594763636589 }, { "label": "completed_group_2", "mean": 0.05962139368057251, "median": 0.05772271752357483, "p05": 0.03670363873243332, "p95": 0.08935610204935074 }, { "label": "completed_group_3", "mean": 0.08040933310985565, "median": 0.07558005303144455, "p05": 0.04971299692988396, "p95": 0.123878113925457 }, { "label": "completed_group_4", "mean": 0.033340416848659515, "median": 0.030976884067058563, "p05": 0.01889173313975334, "p95": 0.056192222982645035 }, { "label": "completed_group_5", "mean": 0.10067726671695709, "median": 0.09931844472885132, "p05": 0.05756794661283493, "p95": 0.14904096722602844 }, { "label": "completed_group_6", "mean": 0.3632875978946686, "median": 0.36857712268829346, "p05": 0.18867944180965424, "p95": 0.5085643529891968 }, { "label": "completed_group_7", "mean": 0.11175162345170975, "median": 0.09184789657592773, "p05": 0.03606988489627838, "p95": 0.2585130035877228 }, { "label": "current_group_partial", "mean": 0.10272546857595444, "median": 0.09300123900175095, "p05": 0.0489085391163826, "p95": 0.18737691640853882 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0009379386901855, "group": 8, "latest_source_mass": 0.13621056079864502, "layer": 32, "max_source_mass": 0.2692604660987854, "mixer_index": 63, "normalized_entropy": 0.9106661009208262, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09139549732208252, "median": 0.07199805974960327, "p05": 0.048967160284519196, "p95": 0.1950986087322235 }, { "label": "completed_group_1", "mean": 0.08807492256164551, "median": 0.08772338181734085, "p05": 0.06209781393408775, "p95": 0.11598332226276398 }, { "label": "completed_group_2", "mean": 0.06457877159118652, "median": 0.06372794508934021, "p05": 0.045170269906520844, "p95": 0.08764317631721497 }, { "label": "completed_group_3", "mean": 0.07007957994937897, "median": 0.06752179563045502, "p05": 0.04661180451512337, "p95": 0.10126036405563354 }, { "label": "completed_group_4", "mean": 0.03462113440036774, "median": 0.033816177397966385, "p05": 0.025839729234576225, "p95": 0.04629064351320267 }, { "label": "completed_group_5", "mean": 0.09155916422605515, "median": 0.08975613117218018, "p05": 0.06288522481918335, "p95": 0.12600551545619965 }, { "label": "completed_group_6", "mean": 0.2692604660987854, "median": 0.2713700532913208, "p05": 0.16808731853961945, "p95": 0.3636854290962219 }, { "label": "completed_group_7", "mean": 0.15421991050243378, "median": 0.14455348253250122, "p05": 0.07559763640165329, "p95": 0.26658111810684204 }, { "label": "current_group_partial", "mean": 0.13621056079864502, "median": 0.12935234606266022, "p05": 0.07068746536970139, "p95": 0.22442279756069183 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.7290174961090088, "group": null, "latest_source_mass": 0.2353290170431137, "layer": null, "max_source_mass": 0.3520195186138153, "mixer_index": null, "normalized_entropy": 0.7869097742412674, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.10267993062734604, "median": 0.10614241659641266, "p05": 0.037625476717948914, "p95": 0.15687566995620728 }, { "label": "completed_group_1", "mean": 0.05008202791213989, "median": 0.04896092414855957, "p05": 0.03316216915845871, "p95": 0.06852905452251434 }, { "label": "completed_group_2", "mean": 0.044380687177181244, "median": 0.0435444712638855, "p05": 0.02717534825205803, "p95": 0.06371284276247025 }, { "label": "completed_group_3", "mean": 0.0326685830950737, "median": 0.03019065037369728, "p05": 0.017900623381137848, "p95": 0.05591970682144165 }, { "label": "completed_group_4", "mean": 0.033161722123622894, "median": 0.032340701669454575, "p05": 0.02229827456176281, "p95": 0.04706215858459473 }, { "label": "completed_group_5", "mean": 0.04359141364693642, "median": 0.04160827398300171, "p05": 0.02683408558368683, "p95": 0.06707148253917694 }, { "label": "completed_group_6", "mean": 0.1060870811343193, "median": 0.09384027123451233, "p05": 0.048520199954509735, "p95": 0.2056540995836258 }, { "label": "completed_group_7", "mean": 0.3520195186138153, "median": 0.3484790027141571, "p05": 0.12008242309093475, "p95": 0.6019007563591003 }, { "label": "completed_group_8", "mean": 0.2353290170431137, "median": 0.22978201508522034, "p05": 0.03305097296833992, "p95": 0.4762387275695801 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "f261d623683eac3593609972ee3faff36fe6f0b7a8891b85f65ae72db324d1ad", "optimizer_state_before": "f261d623683eac3593609972ee3faff36fe6f0b7a8891b85f65ae72db324d1ad", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "41db358e51c17755a85e4f4c84288ab48da8d214d2518c64ab3d3ea8eb9826e1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08832799643278122, 0.17041416466236115, 0.08745641261339188, 0.05030680447816849, 0.12814420461654663, 0.05527547001838684, 0.06746833771467209, 0.04974694177508354, 0.03680809214711189, 0.03558321297168732, 0.051462169736623764, 0.0526844747364521, 0.05210389941930771, 0.03620988130569458, 0.04957791045308113, 0.4858623445034027, 0.04707946628332138, 0.04491531476378441, 0.07858023792505264, 0.09729185700416565, 0.05887215584516525, 0.026784580200910568, 0.041033755987882614, 0.032232560217380524, 0.02874751202762127, 0.022572210058569908, 0.02659173309803009, 0.025225715711712837, 0.041177187114953995, 0.027006076648831367, 0.023083338513970375, 0.016772178933024406 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.076440735749202e-06, "first_to_last_ratio": 0.7397477609382365, "imbalance_abs_log_ratio": 0.30144601448810004, "last_quartile_mean": 1.0917830593371036e-05, "mean": 1.1650229552628844e-05, "normalized": [ 0.7822818865880429, 0.8682868511675877, 0.7339307440237431, 0.6974478440894981, 0.7421854363314211, 0.6291858737475481, 0.5631776493231546, 0.5294481610694797, 0.6111720271757957, 0.5670561579456124, 0.5320779151344551, 0.4996505790449965, 0.4443598163721978, 0.4189457172671887, 0.40342707684344364, 0.4824682206447709, 1.039372314743425, 1.0567625256698632, 1.0003136048176742, 0.9195164682282294, 3.600772368765321, 3.0127235740722966, 2.4106853870817164, 1.9576767978188092, 1.5527140323483644, 1.4071601191923482, 1.1966803370484358, 0.9452267132705859, 0.8812449826684232, 0.5867822018999592, 0.49102686555744746, 0.4362397500481644 ], "peak_layer": 21, "peak_normalized": 3.600772368765321, "population_cv": 0.7485297797974629, "rest_mean": 8.399137136115718e-06, "spike_contrast": 3.4772772641388783, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.920612860179972e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.113763553614262e-06, 1.0115741133631673e-05, 8.550461643608287e-06, 8.125427484628744e-06, 8.646630703879055e-06, 7.330159860430285e-06, 6.56114889352466e-06, 6.168192612676648e-06, 7.120294412743533e-06, 6.606334409298142e-06, 6.198829851200571e-06, 5.821043941978132e-06, 5.176893864700105e-06, 4.88081377625349e-06, 4.700018052972155e-06, 5.6208655223599635e-06, 1.2108926057408098e-05, 1.2311526006669737e-05, 1.1653883120743558e-05, 1.071257793228142e-05, 4.194982466287911e-05, 3.5098921216558665e-05, 2.8085038138669915e-05, 2.2807384084444493e-05, 1.8089474906446412e-05, 1.639373840589542e-05, 1.3941600627731532e-05, 1.101210818887921e-05, 1.0266706340189558e-05, 6.8361473495315295e-06, 5.720575700252084e-06, 5.082293228042545e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.205728376087791e-05, "first_to_last_ratio": 0.7358290958532654, "imbalance_abs_log_ratio": 0.3067573939385225, "last_quartile_mean": 4.356620843282144e-05, "mean": 4.66029975427773e-05, "normalized": [ 0.770903229800344, 0.8488940585533936, 0.7281730770967723, 0.696963880781291, 0.7270770698623731, 0.6281766634906372, 0.5677765947233064, 0.5350777965526399, 0.616871083452953, 0.5727615541231824, 0.5392590563455532, 0.5043753474566816, 0.4510025663848085, 0.42369383655582077, 0.4063875388466082, 0.48098941018748065, 1.0458107275691728, 1.050722619820089, 1.0070247957719578, 0.911755381743329, 3.606423358142869, 3.0176573271540965, 2.418448938919519, 1.9650777400909958, 1.5495404924169884, 1.402027598502842, 1.194110810724892, 0.9430892449785634, 0.8795982640712019, 0.5857415506262685, 0.4894539193353971, 0.43513446591797145 ], "peak_layer": 21, "peak_normalized": 3.606423358142869, "population_cv": 0.7497897507905983, "rest_mean": 3.355908113546859e-05, "spike_contrast": 3.4875849451832757, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011704014614224433, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.592640132410452e-05, 3.956100772484206e-05, 3.393504812265746e-05, 3.248060602345504e-05, 3.3883970900205895e-05, 2.927491550508421e-05, 2.6460091248736717e-05, 2.493622923793737e-05, 2.8748041586368345e-05, 2.669240529939998e-05, 2.513108847779222e-05, 2.350540307816118e-05, 2.1018071493017487e-05, 1.9745402823900804e-05, 1.89388774742838e-05, 2.2415548301069066e-05, 4.873791476711631e-05, 4.896682366961613e-05, 4.693037408287637e-05, 4.2490533814998344e-05, 0.0001680701388977468, 0.00014063187700230628, 0.00011270696995779872, 9.157851309282705e-05, 7.221323176054284e-05, 6.533868872793391e-05, 5.564914317801595e-05, 4.395078576635569e-05, 4.0991915739141405e-05, 2.7297312044538558e-05, 2.281001980009023e-05, 2.0278570445952937e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02842872031033039, "first_to_last_ratio": 0.7358291075435154, "imbalance_abs_log_ratio": 0.30675737805134173, "last_quartile_mean": 0.0386349493637681, "mean": 0.04132800444494933, "normalized": [ 0.7709032599004878, 0.8488941256873797, 0.7281730724499529, 0.6969638541985573, 0.7270771099736086, 0.6281766484573852, 0.567776573083768, 0.5350777977388507, 0.6168710678618747, 0.5727615189938741, 0.5392590137957409, 0.5043753443878958, 0.45100255319043453, 0.4236938111733711, 0.4063875464463718, 0.48098938815921266, 1.0458106889833942, 1.0507225767377182, 1.007024785618777, 0.9117554181054791, 3.606423512151686, 3.0176575247118804, 2.4184488631852656, 1.9650776212623908, 1.549540498537689, 1.4020275638202186, 1.1941108129352203, 0.9430892093010953, 0.8795982916909686, 0.5857415330277667, 0.4894539484537472, 0.43513446597793715 ], "peak_layer": 21, "peak_normalized": 3.606423512151686, "population_cv": 0.7497897765581046, "rest_mean": 0.029760528631784296, "spike_contrast": 3.4875850197496177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1037923738360405, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03185989335179329, 0.0350831001996994, 0.03009393997490406, 0.028804125264286995, 0.03004864603281021, 0.025961287319660187, 0.023465072736144066, 0.022113697603344917, 0.025494050234556198, 0.023671090602874756, 0.02228649891912937, 0.020844826474785805, 0.018639035522937775, 0.017510419711470604, 0.016795186325907707, 0.019878331571817398, 0.04322126880288124, 0.04342426732182503, 0.04161832481622696, 0.037681031972169876, 0.14904628694057465, 0.12471376359462738, 0.0999496653676033, 0.08121273666620255, 0.06403941661119461, 0.05794300138950348, 0.04935021698474884, 0.038975995033979416, 0.036352042108774185, 0.024207528680562973, 0.020228154957294464, 0.017983239144086838 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.3043631952314172e-06, "first_to_last_ratio": 0.4450420118390764, "imbalance_abs_log_ratio": 0.8095865926449729, "last_quartile_mean": 2.9308765476798726e-06, "mean": 2.6055976825034577e-06, "normalized": [ 0.6733728960131794, 0.5812204606869391, 0.5698542538139965, 0.5614717618725491, 0.4098041093483211, 0.40675520855306163, 0.3998664066116846, 0.40245795919812427, 0.45755685594196943, 0.4432237760005925, 0.4415823924604673, 0.4405822214062051, 0.4093562292335019, 0.4106227286093283, 0.4071803565159256, 0.41488511618036494, 0.769918701482759, 0.7873489825883292, 0.8102338457371544, 0.8263187848592529, 3.7447318793466984, 3.357046502905009, 2.8666278561167187, 2.409272840740863, 1.8742389642782162, 1.717309102182846, 1.4585974041085068, 1.1705649823440567, 1.0363341993226067, 0.6723831965836182, 0.5682358642164441, 0.5010441607407099 ], "peak_layer": 21, "peak_normalized": 3.7447318793466984, "population_cv": 0.8923650037371901, "rest_mean": 1.7127541190752204e-06, "spike_contrast": 4.336263356368996, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.426952925015939e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.754538857312582e-06, 1.5144266853894806e-06, 1.4848109231024864e-06, 1.4629695215262473e-06, 1.067784637598379e-06, 1.059840428752068e-06, 1.0418909823783906e-06, 1.0486435257917037e-06, 1.192209083455964e-06, 1.1548628435775754e-06, 1.150586058429326e-06, 1.1479800150482333e-06, 1.0666176422091667e-06, 1.069917630047712e-06, 1.0609481932988274e-06, 1.0810236972247367e-06, 2.0060983842995483e-06, 2.051514684353606e-06, 2.1111434307385935e-06, 2.1530543108383426e-06, 9.757264706422575e-06, 8.74711258802563e-06, 7.469278898497578e-06, 6.277595730352914e-06, 4.883512701781001e-06, 4.474616616789717e-06, 3.8005180158506846e-06, 3.050021405215375e-06, 2.7002699880540604e-06, 1.7519600987725426e-06, 1.4805940509177162e-06, 1.3055195040578838e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.5097264369833283e-05, "first_to_last_ratio": 0.8841933392684389, "imbalance_abs_log_ratio": 0.12307953067826505, "last_quartile_mean": 2.838436262209143e-05, "mean": 3.3539646210556384e-05, "normalized": [ 0.8485407160857603, 0.9216195218127955, 0.8021469941710103, 0.7692195219321104, 0.7681705820049233, 0.6786898940432919, 0.6159832336652391, 0.5819215114063201, 0.6596195359678714, 0.6147763643099993, 0.5770323156855243, 0.5403848244462823, 0.5142674622180825, 0.4828105450759033, 0.4626075097666633, 0.534068298132254, 1.1134406485128994, 1.1076985687998104, 1.0384059475615184, 0.9464901526906941, 3.6165010328459855, 2.9556041233139223, 2.271107394153319, 1.8085505389490877, 1.4615469366449991, 1.285848929384096, 1.0756954067402729, 0.8610270657251896, 0.7574296454091144, 0.5047961437234808, 0.42883702412472346, 0.3951616106968552 ], "peak_layer": 21, "peak_normalized": 3.6165010328459855, "population_cv": 0.7191375453041929, "rest_mean": 2.4703427630743977e-05, "spike_contrast": 3.289228837233073, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.125522654154338e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.8459755412768573e-05, 3.0910792702343315e-05, 2.690372639335692e-05, 2.57993506238563e-05, 2.5764169549802318e-05, 2.276301893289201e-05, 2.0659859728766605e-05, 1.9517441614880227e-05, 2.2123405869933777e-05, 2.06193817575695e-05, 1.9353459720150568e-05, 1.812431582948193e-05, 1.724834874039516e-05, 1.6193294868571684e-05, 1.5515692211920395e-05, 1.791246177162975e-05, 3.734440542757511e-05, 3.715181810548529e-05, 3.482776810415089e-05, 3.1744944863021374e-05, 0.00012129616516176611, 9.912991663441062e-05, 7.617213850608096e-05, 6.065814523026347e-05, 4.901976717519574e-05, 4.312691817176528e-05, 3.60784433723893e-05, 2.887854316213634e-05, 2.540392233640887e-05, 1.6930684068938717e-05, 1.4383042071131058e-05, 1.3253580618766136e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.942075255101372e-05, "first_to_last_ratio": 1.0607277160429414, "imbalance_abs_log_ratio": 0.058955197142597754, "last_quartile_mean": 1.8308895164409478e-05, "mean": 2.3988490454485145e-05, "normalized": [ 0.9265433805953981, 1.002916928598659, 0.8771624828869276, 0.838576400799728, 0.8117319176145469, 0.7227362607416208, 0.665755087924021, 0.6312677145244906, 0.7026744795970906, 0.6544976481800655, 0.6129505366421577, 0.5743798475555982, 0.5940726516503032, 0.5567087586484061, 0.5341993766607235, 0.5982231505815051, 1.1866035009261522, 1.176017666141157, 1.071729859322026, 0.9847732063269867, 3.624279119252218, 2.891470983126283, 2.056678834001149, 1.5981569839009522, 1.3288427129095406, 1.1368223852010402, 0.9591780585040989, 0.8005681249434506, 0.6527448176244227, 0.45705341091844826, 0.40201406892297076, 0.3686696447778616 ], "peak_layer": 21, "peak_normalized": 3.624279119252218, "population_cv": 0.6854481321415927, "rest_mean": 1.8213991131267028e-05, "spike_contrast": 3.0290333624437817, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.517078679986298e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.2226377041079104e-05, 2.405846316833049e-05, 2.104180384776555e-05, 2.0116181985940784e-05, 1.947222335729748e-05, 1.7337351891910657e-05, 1.59704595716903e-05, 1.5143159544095397e-05, 1.6856100046425126e-05, 1.5700410585850477e-05, 1.4703758097311947e-05, 1.3778505490336102e-05, 1.4250906133383978e-05, 1.3354602742765564e-05, 1.2814636647817679e-05, 1.4350470337376464e-05, 2.846482675522566e-05, 2.8210888558533043e-05, 2.5709181500133127e-05, 2.3623222659807652e-05, 8.694098505657166e-05, 6.936202407814562e-05, 4.93366205773782e-05, 3.833737355307676e-05, 3.187693073414266e-05, 2.727065293584019e-05, 2.300923370057717e-05, 1.9204420823371038e-05, 1.565836282679811e-05, 1.0964021385007072e-05, 9.643710654927418e-06, 8.843828254612163e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.355073401209665e-05, "first_to_last_ratio": 0.6873255585991813, "imbalance_abs_log_ratio": 0.37494721458370633, "last_quartile_mean": 9.246089166481397e-05, "mean": 9.620298442314379e-05, "normalized": [ 0.7444695267276318, 0.8149049645336431, 0.7060285527098226, 0.6746516185572761, 0.6906865904744257, 0.6051414541820805, 0.5346022124048582, 0.5142354572810733, 0.6084072519080361, 0.5653159734269397, 0.535822410124007, 0.506437491196043, 0.4288114445296347, 0.40180191593012804, 0.38634873785526624, 0.45058433694504346, 1.02746448873708, 1.018965798576808, 0.9889253452805591, 0.8860995894165328, 3.6016343169500162, 3.0479477917075943, 2.516639360612916, 2.0552564610408948, 1.5982436897490269, 1.4392591156998307, 1.2480975795145037, 1.003440959836939, 0.8890976144939841, 0.5932797044328455, 0.47882480290953644, 0.4385734422550231 ], "peak_layer": 21, "peak_normalized": 3.6016343169500162, "population_cv": 0.7690776651523741, "rest_mean": 6.834074156358838e-05, "spike_contrast": 3.6092540148285686, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000246659095864743, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 7.162019028328359e-05, 7.839628960937262e-05, 6.792205385863781e-05, 6.490349915111437e-05, 6.644611130468547e-05, 5.821641389047727e-05, 5.143032831256278e-05, 4.9470985686639324e-05, 5.853059337823652e-05, 5.4385083785746247e-05, 5.154771497473121e-05, 4.872079807682894e-05, 4.1252940718550235e-05, 3.8654543459415436e-05, 3.716790160979144e-05, 4.334755794843659e-05, 9.884515020530671e-05, 9.802755084820092e-05, 9.513756958767772e-05, 8.524542499799281e-05, 0.00034648797009140253, 0.0002932216739282012, 0.00024210821720771492, 0.00019772180530708283, 0.0001537558127893135, 0.00013846102228853852, 0.00012007071200059727, 9.653401502873749e-05, 8.553384395781904e-05, 5.707527816412039e-05, 4.606437505572103e-05, 4.219207403366454e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6c58c506475022833bc648a96c247244a57dd96d762e84fedda83974a9fa4cd0", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.11756844818592072, 0.05185626819729805, 0.0881219431757927, 0.03964918479323387, 0.022397184744477272, 0.022468235343694687, 0.031668391078710556, 0.045926205813884735, 0.03180550038814545, 0.03912416473031044, 0.06056958809494972, 0.10351218283176422, 0.05620818957686424, 0.0675361230969429, 0.0823286920785904, 0.07062793523073196, 0.0371064618229866, 0.023919809609651566, 0.035764746367931366, 0.07103991508483887, 0.045838613063097, 0.05455324053764343, 0.06383209675550461, 0.07182251662015915, 0.05687069892883301, 0.052836012095212936, 0.06559387594461441, 0.07689779996871948, 0.07740812748670578, 0.08842137455940247, 0.11899886280298233, 0.16321057081222534 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.0315435866396e-06, "first_to_last_ratio": 0.8686691420702204, "imbalance_abs_log_ratio": 0.14079296031078245, "last_quartile_mean": 9.24580280070586e-06, "mean": 1.0551741240760748e-05, "normalized": [ 0.859577091723593, 0.9553086695692968, 0.8084573574612413, 0.7695612835108373, 0.8176461339830406, 0.6924183904282428, 0.6104697187838303, 0.5758263608607518, 0.6429532324009292, 0.6044828265447498, 0.5630258916216904, 0.536585037633965, 0.4709644027945216, 0.4456596693190924, 0.44879673586224333, 0.5476213352179609, 1.1393797585544587, 1.1777628931854884, 1.083279918764496, 1.0241341623776496, 3.41283327136807, 2.7698759447615164, 2.2051169112537026, 1.8283846589145576, 1.5645585305615886, 1.3333451203358413, 1.0503960349342405, 0.9149561789360716, 0.6506532695639259, 0.5429656631634003, 0.4819381667747187, 0.47106537883428756 ], "peak_layer": 21, "peak_normalized": 3.41283327136807, "population_cv": 0.680677072069579, "rest_mean": 7.901781120583311e-06, "spike_contrast": 3.1463192298451848, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4861525889718906e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.07003504835302e-06, 1.0080169886350632e-05, 8.530632840120234e-06, 8.120211532514077e-06, 8.627590432297438e-06, 7.3062196861428674e-06, 6.441518507926958e-06, 6.075970759411575e-06, 6.784276138205314e-06, 6.378346370183863e-06, 5.940903520240681e-06, 5.661906470777467e-06, 4.96949451189721e-06, 4.702485512098065e-06, 4.735587026516441e-06, 5.7783586271398235e-06, 1.2022440387227107e-05, 1.2427449291863013e-05, 1.1430489394115284e-05, 1.080639867723221e-05, 3.601133357733488e-05, 2.9227014238131233e-05, 2.3267823053174652e-05, 1.9292641809443012e-05, 1.6508816770510748e-05, 1.4069112694414798e-05, 1.1083507160947192e-05, 9.654380846768618e-06, 6.865524937893497e-06, 5.7292331803182606e-06, 5.08528682985343e-06, 4.970559984940337e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.186719573022856e-05, "first_to_last_ratio": 0.8636501027700138, "imbalance_abs_log_ratio": 0.14658756584787153, "last_quartile_mean": 3.689827121888811e-05, "mean": 4.217961037511486e-05, "normalized": [ 0.8482760786319284, 0.9347808205670737, 0.803242608202134, 0.7694642925784047, 0.8009222324808606, 0.6911336504850758, 0.615226970004603, 0.5810481970164953, 0.646501974290034, 0.6090268670825646, 0.5690153717762457, 0.5404692649235632, 0.47664103577546924, 0.448947205348827, 0.4514020012299935, 0.547017773515084, 1.1465663590184285, 1.1708024389092253, 1.0904267732668775, 1.0148176111319622, 3.42036094521711, 2.784326838457232, 2.216199364233774, 1.8250690425828833, 1.5605387928833803, 1.3332716055000426, 1.0480080487898067, 0.9133789058697885, 0.6504212469789677, 0.541778233519404, 0.48088076923057743, 0.47003668050218345 ], "peak_layer": 21, "peak_normalized": 3.42036094521711, "population_cv": 0.6827034882268642, "rest_mean": 3.154645087811837e-05, "spike_contrast": 3.157206877049384, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.95986716588959e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.577995448722504e-05, 3.9428690797649324e-05, 3.388046025065705e-05, 3.2455704058520496e-05, 3.3782587706809863e-05, 2.9151748094591312e-05, 2.5950033887056634e-05, 2.450838655931875e-05, 2.726920138229616e-05, 2.5688515961519442e-05, 2.4000846678973176e-05, 2.2796783014200628e-05, 2.0104533177800477e-05, 1.8936418200610206e-05, 1.9039960534428246e-05, 2.3072996555129066e-05, 4.8361722292611375e-05, 4.938399069942534e-05, 4.599377643899061e-05, 4.280461143935099e-05, 0.00014426949201151729, 0.00011744182120310143, 9.347842569695786e-05, 7.698070112382993e-05, 6.582291825907305e-05, 5.623687684419565e-05, 4.420457116793841e-05, 3.852596637443639e-05, 2.7434514777269214e-05, 2.2851994799566455e-05, 2.028336348303128e-05, 1.9825964045594446e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02826014719903469, "first_to_last_ratio": 0.863650085818606, "imbalance_abs_log_ratio": 0.14658758547550527, "last_quartile_mean": 0.03272175579331815, "mean": 0.037405300070531666, "normalized": [ 0.8482760062421019, 0.9347808255690103, 0.8032425824902223, 0.7694642654684261, 0.8009222247907771, 0.6911336564434044, 0.6152269813998814, 0.5810482026183381, 0.6465019891547976, 0.6090268951969023, 0.5690153774396436, 0.5404693059390998, 0.476641019679117, 0.44894721167248924, 0.45140201935344615, 0.5470177669830626, 1.1465664364462826, 1.1708023901135867, 1.0904266972771928, 1.0148176048594157, 3.420360920331411, 2.7843268581630904, 2.2161993983606085, 1.8250690648858443, 1.5605389254466793, 1.3332715485871998, 1.0480080268879794, 0.9133788783268157, 0.6504212061158645, 0.5417782512035506, 0.4808807753553657, 0.47003668719839264 ], "peak_layer": 21, "peak_normalized": 3.420360920331411, "population_cv": 0.6827034910812323, "rest_mean": 0.02797570769433622, "spike_contrast": 3.157206954942155, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08832509890198707, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03173001855611801, 0.03496575728058815, 0.030045529827475548, 0.028782041743397713, 0.029958736151456833, 0.025852061808109283, 0.02301274985074997, 0.02173428237438202, 0.024182600900530815, 0.022780833765864372, 0.0212841909378767, 0.02021641656756401, 0.01782890036702156, 0.016793005168437958, 0.016884827986359596, 0.020461363717913628, 0.04288766160607338, 0.043794214725494385, 0.04078773781657219, 0.03795955702662468, 0.1279396265745163, 0.10414858162403107, 0.0828976035118103, 0.06826725602149963, 0.05837242677807808, 0.04987142235040665, 0.039201054722070694, 0.03416521102190018, 0.024329200387001038, 0.020265378057956696, 0.017987489700317383, 0.017581863328814507 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2999315259776267e-06, "first_to_last_ratio": 0.5262833924647308, "imbalance_abs_log_ratio": 0.641915442383951, "last_quartile_mean": 2.4700219398710033e-06, "mean": 2.3013876315758353e-06, "normalized": [ 0.7510790889350714, 0.657054866040563, 0.6465905227557444, 0.6364494482049028, 0.464747216363479, 0.46266301233609036, 0.448491551252894, 0.45169899557263793, 0.49533187678596646, 0.4887005700041262, 0.4851101332493999, 0.48095555698641873, 0.4488186731288716, 0.4438844284018987, 0.4569450978053562, 0.47536237557813144, 0.861237602967154, 0.8798991600628995, 0.8809450411197095, 0.9018750120700136, 3.4686861617847566, 3.081635479089349, 2.692581952941867, 2.3530557260816174, 1.9593302158594676, 1.6585873452103246, 1.316811871407005, 1.144956403072183, 0.7621399776403528, 0.6320030705342806, 0.5650087375706647, 0.5473628291868035 ], "peak_layer": 21, "peak_normalized": 3.4686861617847566, "population_cv": 0.8118793877537595, "rest_mean": 1.57216401964467e-06, "spike_contrast": 3.9685395785957933, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.239195136004127e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.72852412561042e-06, 1.512137941972469e-06, 1.4880554317642236e-06, 1.4647168882220285e-06, 1.0695634955482092e-06, 1.0647669341778965e-06, 1.0321529089196702e-06, 1.039534481606097e-06, 1.1399506547604688e-06, 1.1246894473515567e-06, 1.1164264606122742e-06, 1.1068651701862109e-06, 1.0329057431590627e-06, 1.021550133373239e-06, 1.0516077963984571e-06, 1.0939930916720186e-06, 1.982041567316628e-06, 2.024989044002723e-06, 2.027396021730965e-06, 2.0755639980052365e-06, 7.982791430549696e-06, 7.092037776601501e-06, 6.19667480350472e-06, 5.41529334441293e-06, 4.50917832495179e-06, 3.817052402155241e-06, 3.0304945539683104e-06, 2.6349885047238786e-06, 1.7539795180709916e-06, 1.4544840496455436e-06, 1.3003041203774046e-06, 1.2596940450748662e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.497316063454491e-05, "first_to_last_ratio": 1.0253055639724644, "imbalance_abs_log_ratio": 0.024990679356707225, "last_quartile_mean": 2.4356798121516476e-05, "mean": 3.0721413736500836e-05, "normalized": [ 0.923683121360003, 1.0034279851985808, 0.8748807920505384, 0.8394613337717378, 0.8363423131230837, 0.7383758864764528, 0.6615002397989881, 0.6254563024059205, 0.6885636855662547, 0.64879661636469, 0.6045999907493074, 0.5741721615335839, 0.5386709353912388, 0.5062958349979282, 0.5053995265849414, 0.597134898616305, 1.2072446578739298, 1.220368251949879, 1.111346881203051, 1.0425813513547888, 3.4234834365844424, 2.7178133929240076, 2.0856692963667, 1.6821068182789498, 1.431433074359949, 1.2175281064911556, 0.9576212846278308, 0.8451805251137674, 0.5586565169962346, 0.4724016049341192, 0.4327639669778174, 0.42703920997382316 ], "peak_layer": 21, "peak_normalized": 3.4234834365844424, "population_cv": 0.6561034075590804, "rest_mean": 2.3506994895999218e-05, "spike_contrast": 2.964193244754933, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.967927547520957e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.8376851332723163e-05, 3.082672628806904e-05, 2.687757478270214e-05, 2.5789438950596377e-05, 2.569361822679639e-05, 2.268395110149868e-05, 2.032222255365923e-05, 1.921490184031427e-05, 2.1153649868210778e-05, 1.993194928218145e-05, 1.8574166460894048e-05, 1.763938053045422e-05, 1.6548732673982158e-05, 1.5554123820038512e-05, 1.5526587958447635e-05, 1.8344828276894987e-05, 3.70882626157254e-05, 3.749143797904253e-05, 3.414214734220877e-05, 3.2029573048930615e-05, 0.00010517425107536837, 8.349506970262155e-05, 6.407470937119797e-05, 5.1676699513336644e-05, 4.397564771352336e-05, 3.740418469533324e-05, 2.9419479687931016e-05, 2.5965140594053082e-05, 1.7162717995233834e-05, 1.451284515496809e-05, 1.3295120879774913e-05, 1.3119248251314275e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9352088997948158e-05, "first_to_last_ratio": 1.206212898548031, "imbalance_abs_log_ratio": 0.18748561551764262, "last_quartile_mean": 1.6043676055232936e-05, "mean": 2.226409807803975e-05, "normalized": [ 0.9991480788422865, 1.0765252138266244, 0.9446533114437263, 0.905168326838966, 0.8728412921796674, 0.7752388554010682, 0.7090929337562585, 0.6709804778017834, 0.7282270427975684, 0.6827339620063341, 0.6396496998383137, 0.608460304703459, 0.6143468304484543, 0.5792646484687448, 0.5732065515361786, 0.6525851461240976, 1.266901747363313, 1.2797303701596927, 1.1381896960520796, 1.069494553362227, 3.415424819879391, 2.644513480032256, 1.8913997913758187, 1.49736286595504, 1.261891215081716, 1.0849304046051615, 0.8879356350587702, 0.8045755676621354, 0.5007272020530341, 0.43401113301181293, 0.3983632971667951, 0.3924255451672251 ], "peak_layer": 21, "peak_normalized": 3.415424819879391, "population_cv": 0.6261266628879907, "rest_mean": 1.755516532958373e-05, "spike_contrast": 2.716712376347249, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.769233491970226e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 2.2245130821829662e-05, 2.396786294411868e-05, 2.1031853975728154e-05, 2.015275640587788e-05, 1.9433024135651067e-05, 1.725999391055666e-05, 1.5787314623594284e-05, 1.493877516622888e-05, 1.6213318303925917e-05, 1.5200455891317688e-05, 1.4241223652788904e-05, 1.3546819900511764e-05, 1.3677878087037243e-05, 1.2896804946649354e-05, 1.2761926882376429e-05, 1.452921969757881e-05, 2.820642475853674e-05, 2.849204247468151e-05, 2.5340767024317756e-05, 2.3811331629985943e-05, 7.604135316796601e-05, 5.8877707488136366e-05, 4.211031045997515e-05, 3.33374337060377e-05, 2.809486977639608e-05, 2.415499693597667e-05, 1.976908606593497e-05, 1.791314934962429e-05, 1.1148239536851179e-05, 9.662866432336159e-06, 8.86919951881282e-06, 8.737000825931318e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.326739594442188e-05, "first_to_last_ratio": 0.8019224850951497, "imbalance_abs_log_ratio": 0.22074332778778838, "last_quartile_mean": 7.88946526881773e-05, "mean": 8.741268038647831e-05, "normalized": [ 0.8172201386791746, 0.8930527530237261, 0.7770273032061067, 0.7443277001726077, 0.7577746159675975, 0.6634374473413018, 0.577056141658992, 0.5603298181762961, 0.6302796253573144, 0.6008518970750966, 0.5697352095332306, 0.5465079619158248, 0.4488951312608189, 0.42194694599696664, 0.4248763842791633, 0.5137640768693087, 1.120298638062262, 1.1322557804193147, 1.0880678250613065, 0.9947818548229707, 3.4370126991938204, 2.832083111262142, 2.310584831224533, 1.9174011752223596, 1.605064988302073, 1.3860757842433205, 1.1026642419836798, 0.9897246325205491, 0.6559259444095199, 0.5273199913410769, 0.48280183533337045, 0.4708535160841745 ], "peak_layer": 21, "peak_normalized": 3.4370126991938204, "population_cv": 0.7050175147633709, "rest_mean": 6.441943265532177e-05, "spike_contrast": 3.2843539505659556, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002115762181347236, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 7.143540278775617e-05, 7.806413486832753e-05, 6.792203930672258e-05, 6.506367935799062e-05, 6.623911031056195e-05, 5.7992845540866256e-05, 5.044202407589182e-05, 4.8979931307258084e-05, 5.509443144546822e-05, 5.252207483863458e-05, 4.980208177585155e-05, 4.777172580361366e-05, 3.9239126635948196e-05, 3.688351353048347e-05, 3.713958358275704e-05, 4.490949504543096e-05, 9.792830678634346e-05, 9.897351264953613e-05, 9.511092503089458e-05, 8.69565483299084e-05, 0.00030043849255889654, 0.00024755997583270073, 0.00020197441335767508, 0.00016760517610237002, 0.00014030303282197565, 0.00012116059951949865, 9.638683695811778e-05, 8.651448297314346e-05, 5.733624493586831e-05, 4.6094453864498064e-05, 4.220300252200104e-05, 4.115856791031547e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90bf4c661b8e06ef22fe9277f79a81ef01617138f2a24fe93a0093039186433f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08832799643278122, 0.2790028154850006, 0.315830796957016, 0.3664044439792633, 0.12814420461654663, 0.14859555661678314, 0.17815031111240387, 0.19290222227573395, 0.03680809214711189, 0.06554166972637177, 0.11251214146614075, 0.16864995658397675, 0.05210389941930771, 0.09766767174005508, 0.16688434779644012, 0.6339711546897888, 0.04707946628332138, 0.084575355052948, 0.13482175767421722, 0.19051826000213623, 0.05887215584516525, 0.08281512558460236, 0.11754488199949265, 0.14818532764911652, 0.02874751202762127, 0.07124026119709015, 0.10666517913341522, 0.14956052601337433, 0.041177187114953995, 0.09537366032600403, 0.15850447118282318, 0.25068944692611694 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.076430106029875e-06, "first_to_last_ratio": 0.7397413644921954, "imbalance_abs_log_ratio": 0.3014546613188623, "last_quartile_mean": 1.0917910628904792e-05, "mean": 1.1650247159877836e-05, "normalized": [ 0.7822789087791068, 0.8682934236309049, 0.7339372072758686, 0.6974498346185574, 0.7421498092330202, 0.629192183036434, 0.5631748074843903, 0.5294525913634219, 0.6111628284442171, 0.5670595555698634, 0.5320827317867908, 0.49964759901627454, 0.444357193138454, 0.41894363987490607, 0.40343368829278586, 0.4824716290086071, 1.0393637959957764, 1.0567609285637538, 1.0003416021830591, 0.9195068034895976, 3.60074631727536, 3.012709340630306, 2.4107007920054846, 1.9576641588898815, 1.5527213659541896, 1.4071726690364275, 1.1966858667382718, 0.9452322326544652, 0.8812437288920071, 0.5867980603612143, 0.4910267089580212, 0.4362379978185813 ], "peak_layer": 21, "peak_normalized": 3.60074631727536, "population_cv": 0.7485264383518611, "rest_mean": 8.399162854604785e-06, "spike_contrast": 3.477263498033052, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9206102408352307e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.113742635236122e-06, 1.0115832992596552e-05, 8.550549864594359e-06, 8.125462954922114e-06, 8.646228707220871e-06, 7.33024444343755e-06, 6.561125701409765e-06, 6.168253548821667e-06, 7.120198006305145e-06, 6.606383976759389e-06, 6.198895334819099e-06, 5.821018021379132e-06, 5.176871127332561e-06, 4.880796950601507e-06, 4.700102181232069e-06, 5.6209137255791575e-06, 1.210884511237964e-05, 1.2311526006669737e-05, 1.1654226909740828e-05, 1.0712481525843032e-05, 4.194958455627784e-05, 3.5098808439215645e-05, 2.8085260055377148e-05, 2.2807271307101473e-05, 1.8089587683789432e-05, 1.6393909390899353e-05, 1.3941686120233499e-05, 1.1012189133907668e-05, 1.026670724968426e-05, 6.83634243614506e-06, 5.720582521462347e-06, 5.08228049511672e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.205732150490803e-05, "first_to_last_ratio": 0.735826267840192, "imbalance_abs_log_ratio": 0.30676123724741555, "last_quartile_mean": 4.356642716629722e-05, "mean": 4.660306160531036e-05, "normalized": [ 0.7709022481473525, 0.8488925013124051, 0.728178711481676, 0.6969576144155101, 0.727058115881797, 0.6281832549972298, 0.5677856111509606, 0.5350832279959402, 0.6168642246183335, 0.5727581516676233, 0.5392630769055691, 0.5043755908780991, 0.4510033125216739, 0.4236961034310519, 0.40639369363456396, 0.48099062251322566, 1.0458012494557996, 1.050717740676697, 1.0070350428730397, 0.9117479614221499, 3.6063815548246874, 3.0176519299457474, 2.418472936497765, 1.9650819083638658, 1.5495511646994138, 1.402015679141591, 1.1941147897920268, 0.9430952864984172, 0.8796006458403054, 0.5857526110303165, 0.4894569545079926, 0.4351364828771731 ], "peak_layer": 21, "peak_normalized": 3.6063815548246874, "population_cv": 0.749785809803623, "rest_mean": 3.355914170107873e-05, "spike_contrast": 3.4875811226244497, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011704022908816114, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.5926404962083325e-05, 3.956098953494802e-05, 3.3935357350856066e-05, 3.2480358640896156e-05, 3.3883134165080264e-05, 2.9275262932060286e-05, 2.6460547815077007e-05, 2.493651663826313e-05, 2.87477614620002e-05, 2.6692283427109942e-05, 2.513131039449945e-05, 2.3505446733906865e-05, 2.101813515764661e-05, 1.9745535610127263e-05, 1.893919034046121e-05, 2.2415635612560436e-05, 4.873754005529918e-05, 4.896666359854862e-05, 4.6930916141718626e-05, 4.249024641467258e-05, 0.00016806842177174985, 0.00014063181879464537, 0.0001127082432503812, 9.157883323496208e-05, 7.22138283890672e-05, 6.53382230666466e-05, 5.564940511249006e-05, 4.395112773636356e-05, 4.099208308616653e-05, 2.7297865017317235e-05, 2.2810192604083568e-05, 2.0278692318242975e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028428752906620502, "first_to_last_ratio": 0.7358262440829839, "imbalance_abs_log_ratio": 0.30676126953385263, "last_quartile_mean": 0.038635144010186195, "mean": 0.041328060964588076, "normalized": [ 0.7709022056247067, 0.8488925140550104, 0.7281787018637822, 0.6969575828113492, 0.7270581328057514, 0.6281832258802316, 0.5677856218053148, 0.5350832405294478, 0.6168642299621969, 0.5727581216485953, 0.5392630987758338, 0.5043756010763449, 0.4510033335681608, 0.42369611619969416, 0.4063937060687683, 0.4809906232945811, 1.0458012363337954, 1.050717714484636, 1.0070350364208112, 0.9117480417171357, 3.606381442591284, 3.0176517752977294, 2.4184729581554, 1.965081964731739, 1.549551179216697, 1.4020157310844736, 1.1941147685354327, 0.9430953109844836, 0.8796006943450475, 0.5857526303889504, 0.4894569748028903, 0.43513648493972445 ], "peak_layer": 21, "peak_normalized": 3.606381442591284, "population_cv": 0.7497857870552702, "rest_mean": 0.029760582648493624, "spike_contrast": 3.4875810429320246, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10379244387149811, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03185989335179329, 0.03508308157324791, 0.030094213783740997, 0.028803905472159386, 0.030047902837395668, 0.02596159465610981, 0.023465478792786598, 0.022113952785730362, 0.02549380250275135, 0.0236709825694561, 0.02228669822216034, 0.02084486559033394, 0.0186390932649374, 0.017510538920760155, 0.016795463860034943, 0.019878409802913666, 0.04322093725204468, 0.043424125760793686, 0.04161880537867546, 0.03768077865242958, 0.14904475212097168, 0.12471369653940201, 0.09995079785585403, 0.08121302723884583, 0.06403994560241699, 0.05794259160757065, 0.049350447952747345, 0.03897630050778389, 0.036352191120386124, 0.02420802041888237, 0.0202283076941967, 0.017983347177505493 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.3043483590990945e-06, "first_to_last_ratio": 0.44502800577020923, "imbalance_abs_log_ratio": 0.8096180644811009, "last_quartile_mean": 2.930935451672667e-06, "mean": 2.605597909877133e-06, "normalized": [ 0.6733235333569506, 0.5812250349347619, 0.5698763690235182, 0.5614353676156453, 0.4097737931419157, 0.4067861516118697, 0.3999128395484859, 0.40242406582797585, 0.45755336910444566, 0.4431962056791131, 0.44158440461941995, 0.4405653847296191, 0.40935929136700205, 0.41062613968643547, 0.4071873893299861, 0.4148719468145246, 0.769932770989047, 0.7873496119897289, 0.8102137044210286, 0.8263661841129266, 3.7447615712230267, 3.356866796136387, 2.866602125013433, 2.4093180075356178, 1.8743363613537902, 1.7172763157635853, 1.4586587976158965, 1.1705806748956349, 1.0363602879478928, 0.6724110186074196, 0.5682209361980447, 0.5010435498048719 ], "peak_layer": 21, "peak_normalized": 3.7447615712230267, "population_cv": 0.8923578131661756, "rest_mean": 1.7127574707316213e-06, "spike_contrast": 4.3362451533143265, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.426936281262897e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.7544103911859565e-06, 1.514438736194279e-06, 1.484868676016049e-06, 1.4628748203904252e-06, 1.0677057389330002e-06, 1.0599211464068503e-06, 1.0420120588605641e-06, 1.0485553048056317e-06, 1.192200102195784e-06, 1.154791107182973e-06, 1.1505914017106988e-06, 1.1479362456157105e-06, 1.066625713974645e-06, 1.069926611307892e-06, 1.0609666105665383e-06, 1.0809894774865825e-06, 2.00613521883497e-06, 2.0515165033430094e-06, 2.1110911347932415e-06, 2.1531780021177838e-06, 9.757342922966927e-06, 8.746645107748918e-06, 7.469212505384348e-06, 6.2777139646641444e-06, 4.8837669055501465e-06, 4.4745315790351015e-06, 3.800678314291872e-06, 3.0500625598506304e-06, 2.7003382001566933e-06, 1.7520327446618467e-06, 1.4805552837060532e-06, 1.3055180261289934e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.509732189537317e-05, "first_to_last_ratio": 0.8841928656854868, "imbalance_abs_log_ratio": 0.12308006628858677, "last_quartile_mean": 2.8384442884998862e-05, "mean": 3.3539693617967714e-05, "normalized": [ 0.848541794522451, 0.9216211477618174, 0.8021519345590349, 0.7692168076440962, 0.7681649405673239, 0.6786894770735635, 0.6159871355761651, 0.581923997146414, 0.6596177901053007, 0.614775007235569, 0.5770344286988468, 0.5403836267561264, 0.5142676030584072, 0.48281214046187954, 0.46261206234129265, 0.5340701464704227, 1.1134364714672835, 1.1076980877777178, 1.0384077338398015, 0.946489031789269, 3.6164685871229025, 2.9556055859795665, 2.2711256606434183, 1.8085464640638436, 1.461551487329098, 1.285844508644236, 1.0756934524037012, 0.8610315974843117, 0.7574319915412695, 0.5048016671103398, 0.4288378822917964, 0.39515975053273394 ], "peak_layer": 21, "peak_normalized": 3.6164685871229025, "population_cv": 0.7191344912178012, "rest_mean": 2.4703477754007542e-05, "spike_contrast": 3.289223488792826, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.125525928335265e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.845983181032352e-05, 3.0910890927771106e-05, 2.6903930120170116e-05, 2.5799296054174192e-05, 2.576401675469242e-05, 2.2763037122786045e-05, 2.0660019799834117e-05, 1.9517552573233843e-05, 2.2123378585092723e-05, 2.0619365386664867e-05, 1.935355794557836e-05, 1.81243012775667e-05, 1.7248377844225615e-05, 1.6193371266126633e-05, 1.5515866834903136e-05, 1.791254908312112e-05, 3.734431811608374e-05, 3.715185448527336e-05, 3.4827877243515104e-05, 3.174495213897899e-05, 0.00012129524839110672, 9.913010580930859e-05, 7.617285882588476e-05, 6.065809429856017e-05, 4.901998909190297e-05, 4.312683086027391e-05, 3.607842882047407e-05, 2.8878735975013115e-05, 2.5404036932741292e-05, 1.6930893252720125e-05, 1.4383091183844954e-05, 1.3253536963020451e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9421660454099765e-05, "first_to_last_ratio": 1.0607859196904783, "imbalance_abs_log_ratio": 0.059010067068080395, "last_quartile_mean": 1.8308746462025738e-05, "mean": 2.3987808219771978e-05, "normalized": [ 0.9266617896291613, 1.0030511591186864, 0.8771969847201996, 0.8385340512941075, 0.8118692035934721, 0.7226894791926554, 0.6658428001735601, 0.6313316969985058, 0.7028068439710677, 0.6545495519368416, 0.6129768036688611, 0.5743184579423999, 0.5940844670146944, 0.5568005354338362, 0.5341777923418284, 0.598202628858524, 1.1864309920816327, 1.1759385817920434, 1.0718528525675306, 0.9849245133243942, 3.623573548579859, 2.8913148104249835, 2.0566096304938517, 1.5982435366367922, 1.3288207525445885, 1.1368754947964976, 0.9592605424407418, 0.8006210740556025, 0.6527473063645078, 0.4570466562772806, 0.40203858319370883, 0.36860687853758356 ], "peak_layer": 21, "peak_normalized": 3.623573548579859, "population_cv": 0.6853417288273663, "rest_mean": 1.8214242825504494e-05, "spike_contrast": 3.0286771663970296, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.516506134881638e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.222858529421501e-05, 2.4060998839559034e-05, 2.1042033040430397e-05, 2.011459400819149e-05, 1.947496275533922e-05, 1.733573662932031e-05, 1.5972109395079315e-05, 1.514426367066335e-05, 1.685879578872118e-05, 1.5701209122198634e-05, 1.470397000957746e-05, 1.3776641026197467e-05, 1.425078426109394e-05, 1.3356424460653216e-05, 1.2813754437956959e-05, 1.4349569937621709e-05, 2.8459879104048014e-05, 2.820818917825818e-05, 2.5711400667205453e-05, 2.3626180336577818e-05, 8.692158735357225e-05, 6.935630517546088e-05, 4.933355739922263e-05, 3.833835944533348e-05, 3.187549737049267e-05, 2.727115133893676e-05, 2.301055792486295e-05, 1.920514478115365e-05, 1.565797720104456e-05, 1.0963547538267449e-05, 9.64402443059953e-06, 8.842071110848337e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.355676441671676e-05, "first_to_last_ratio": 0.6873951646261188, "imbalance_abs_log_ratio": 0.3748459488854143, "last_quartile_mean": 9.246030185749987e-05, "mean": 9.619895513424126e-05, "normalized": [ 0.7445812595177274, 0.8150771297110402, 0.7062879022236439, 0.6744850418983919, 0.6907271675605413, 0.6051874865339609, 0.5348077152621196, 0.5142895188349452, 0.60851218903597, 0.5653097004083167, 0.535848710379822, 0.5063380285002478, 0.4287242733552982, 0.4018182537611802, 0.38641419591620674, 0.45067226392677057, 1.0274243261167273, 1.0190208819675328, 0.9889455130861956, 0.8859005728221988, 3.6013359024505665, 3.047775942336502, 2.5162213793615953, 2.0552047393489516, 1.5986593070507766, 1.4391617767539748, 1.2481945560010297, 1.0031304566212418, 0.8891777390557095, 0.5933080709297001, 0.47895592769362005, 0.4385020715774952 ], "peak_layer": 21, "peak_normalized": 3.6013359024505665, "population_cv": 0.7690005057883333, "rest_mean": 6.833974745219436e-05, "spike_contrast": 3.6090077270160457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00024663867661729456, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 7.16279391781427e-05, 7.84095682320185e-05, 6.79441582178697e-05, 6.488475628430024e-05, 6.644723180215806e-05, 5.821840386488475e-05, 5.144794340594672e-05, 4.947411434841342e-05, 5.853823677171022e-05, 5.438220250653103e-05, 5.154808604856953e-05, 4.870918928645551e-05, 4.124282713746652e-05, 3.8654496165690944e-05, 3.717264189617708e-05, 4.335420089773834e-05, 9.883714665193111e-05, 9.802874410524964e-05, 9.513552504358813e-05, 8.522270945832133e-05, 0.0003464447509031743, 0.00029319286113604903, 0.00024205786758102477, 0.0001977085485123098, 0.00015378935495391488, 0.00013844585919287056, 0.00012007501209154725, 9.650010179029778e-05, 8.553796942578629e-05, 5.707561649614945e-05, 4.607505979947746e-05, 4.218344110995531e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "25421e314b847c78a706fbfc95b3ae6bf0cea5ddc086f201de56fb1ddc434ca3", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.15143299102783203, 0.30262914299964905, 0.3626292943954468, 0.38562148809432983, 0.13662196695804596, 0.15442071855068207, 0.18618692457675934, 0.2074802666902542, 0.04909220710396767, 0.0881529152393341, 0.14954733848571777, 0.23951733112335205, 0.08217982947826385, 0.14409078657627106, 0.22605396807193756, 0.6592875719070435, 0.06389845162630081, 0.09458807110786438, 0.14630787074565887, 0.21908092498779297, 0.07773751765489578, 0.10736925154924393, 0.1424422264099121, 0.17246152460575104, 0.06572399288415909, 0.10123347491025925, 0.14551502466201782, 0.19983991980552673, 0.09023923426866531, 0.15600618720054626, 0.24885882437229156, 0.3860291540622711 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.031635957195249e-06, "first_to_last_ratio": 0.8686801313140773, "imbalance_abs_log_ratio": 0.1407803097240232, "last_quartile_mean": 9.245792170986533e-06, "mean": 1.055174838882067e-05, "normalized": [ 0.8595768541961558, 0.9553206928959411, 0.8084617228322365, 0.7695638651625307, 0.8176598882474133, 0.6924422710967179, 0.6104757697647815, 0.57582984949779, 0.6429573651142346, 0.604473194319946, 0.5630245620804605, 0.5365777355393115, 0.4709580501875008, 0.44566609052837647, 0.4487992331310253, 0.5476071301474617, 1.1393753665683581, 1.1777433050985566, 1.0832692726387225, 1.0241332100182778, 3.4128037221928054, 2.7698985473907003, 2.205139206917057, 1.828377559139235, 1.564558332621775, 1.3333393040458241, 1.05039290994071, 0.9149440953502781, 0.6506508463364419, 0.5429766729169746, 0.48193486661167245, 0.47106850747072765 ], "peak_layer": 21, "peak_normalized": 3.4128037221928054, "population_cv": 0.6806759200046126, "rest_mean": 7.901783326950087e-06, "spike_contrast": 3.146322633035019, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.486155972292181e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.070038686331827e-06, 1.0080303582071792e-05, 8.530684681318235e-06, 8.12024427432334e-06, 8.627741408417933e-06, 7.306476618396118e-06, 6.441586720029591e-06, 6.076011686673155e-06, 6.784324341424508e-06, 6.378249054250773e-06, 5.940893515798962e-06, 5.661833256453974e-06, 4.969430847268086e-06, 4.702556452684803e-06, 4.735616585094249e-06, 5.778212653240189e-06, 1.2022402188449632e-05, 1.2427251022018027e-05, 1.143038480222458e-05, 1.0806395948748104e-05, 3.601104617700912e-05, 2.9227272534626536e-05, 2.326807407371234e-05, 1.929257996380329e-05, 1.6508825865457766e-05, 1.4069060853216797e-05, 1.1083481695095543e-05, 9.654259883973282e-06, 6.865504019515356e-06, 5.729353233618895e-06, 5.085255452286219e-06, 4.970596364728408e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.186757840012433e-05, "first_to_last_ratio": 0.8636564556008468, "imbalance_abs_log_ratio": 0.146580210082559, "last_quartile_mean": 3.689844288601307e-05, "mean": 4.2179749868864747e-05, "normalized": [ 0.8482764645044092, 0.9347954102594115, 0.8032484042203742, 0.7694648528419803, 0.8009406285876599, 0.6911447334785021, 0.6152281265998635, 0.5810488197746441, 0.646507771172811, 0.6090191173677569, 0.569016724323795, 0.5404629494292167, 0.4766369582304063, 0.4489483081063157, 0.4514009827584627, 0.5470082020108817, 1.146557047216301, 1.170778557053383, 1.090417647125437, 1.014806923801809, 3.420348943656972, 2.7843447126541196, 2.216221532283493, 1.825052484420704, 1.5605363919661974, 1.3332759073938238, 1.048016916561615, 0.913369157756135, 0.6504165515961319, 0.5417902416971032, 0.48087879077488155, 0.470039740375403 ], "peak_layer": 21, "peak_normalized": 3.420348943656972, "population_cv": 0.6827037752697015, "rest_mean": 3.154654101914881e-05, "spike_contrast": 3.1572107254761765, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.959907765733078e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.57800890924409e-05, 3.942943658330478e-05, 3.388081677258015e-05, 3.245583502575755e-05, 3.3783475373638794e-05, 2.915231198130641e-05, 2.5950168492272496e-05, 2.450849387969356e-05, 2.7269536076346412e-05, 2.568827403592877e-05, 2.400098310317844e-05, 2.2796592020313255e-05, 2.010442767641507e-05, 1.8936527339974418e-05, 1.9039980543311685e-05, 2.3072669137036428e-05, 4.836148946196772e-05, 4.9383146688342094e-05, 4.5993543608346954e-05, 4.280430221115239e-05, 0.00014426946290768683, 0.00011744296352844685, 9.347966988570988e-05, 7.698025729041547e-05, 6.582303467439488e-05, 5.6237244280055165e-05, 4.4205091398907825e-05, 3.852568261208944e-05, 2.7434407456894405e-05, 2.285257687617559e-05, 2.028334711212665e-05, 1.9826158677460626e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028260487830266356, "first_to_last_ratio": 0.8636564890006067, "imbalance_abs_log_ratio": 0.1465801714100533, "last_quartile_mean": 0.032721907598897815, "mean": 0.037405423703603446, "normalized": [ 0.848276489049398, 0.9347954633352487, 0.803248442735902, 0.7694648593773151, 0.8009406413292095, 0.6911447672537372, 0.6152281348927102, 0.5810488217298512, 0.6465077699087762, 0.6090191058798913, 0.5690167334680125, 0.5404629383268489, 0.47663695447049736, 0.4489483172026723, 0.45140097553658615, 0.5470081907747307, 1.1465570696285177, 1.1707785023145068, 1.0904177151979917, 1.0148068808397375, 3.4203488185556594, 2.7843447444483567, 2.216221552644854, 1.8250524758566367, 1.5605363569241573, 1.3332759059523371, 1.048016912432173, 0.9133691867299689, 0.6504165665258815, 0.5417902042388995, 0.48087878757067437, 0.47003971486826046 ], "peak_layer": 21, "peak_normalized": 3.4203488185556594, "population_cv": 0.6827037640207692, "rest_mean": 0.02797578799503821, "spike_contrast": 3.1572106760860907, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08832545652985573, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03173014149069786, 0.034966420382261276, 0.030045848339796066, 0.028782159090042114, 0.029959524050354958, 0.025852562859654427, 0.02301286906003952, 0.02173437736928463, 0.024182897061109543, 0.02278061769902706, 0.0212843120098114, 0.02021624520421028, 0.0178288072347641, 0.016793102025985718, 0.01688484475016594, 0.020461073145270348, 0.042887452989816666, 0.043793465942144394, 0.04078753665089607, 0.03795928135514259, 0.1279395967721939, 0.10414959490299225, 0.08289870619773865, 0.068266861140728, 0.05837252363562584, 0.04987175017595291, 0.0392015166580677, 0.03416496142745018, 0.024329107254743576, 0.020265892148017883, 0.01798747479915619, 0.017582034692168236 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2999394698454125e-06, "first_to_last_ratio": 0.5262796475240541, "imbalance_abs_log_ratio": 0.6419225582345641, "last_quartile_mean": 2.4700546106259935e-06, "mean": 2.301371839763533e-06, "normalized": [ 0.7510621611549569, 0.6571077368974771, 0.6465980717778745, 0.6364463561264533, 0.46473272036308516, 0.46266426050499954, 0.44850722565952067, 0.4517147907876578, 0.49535804892346047, 0.4887073813963764, 0.4851234901505736, 0.4809132614348466, 0.44883904274174174, 0.4438839175271594, 0.45695317328219487, 0.4753797657574617, 0.8612276060345527, 0.8798755580967531, 0.8808719479357286, 0.901864305995852, 3.468670048732778, 3.0815001270843183, 2.6925377905168015, 2.353088273209896, 1.9593845634770406, 1.6586749992669176, 1.3167743728360517, 1.144966828441851, 0.7621392794267813, 0.6320454943721241, 0.5650235813213862, 0.547363818765327 ], "peak_layer": 21, "peak_normalized": 3.468670048732778, "population_cv": 0.8118673460429436, "rest_mean": 1.5721624996095435e-06, "spike_contrast": 3.9684843507872736, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.2391022765950765e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.7284733075939585e-06, 1.5122492413865984e-06, 1.488062594035e-06, 1.4646997215095325e-06, 1.0695227956603048e-06, 1.0647625003912253e-06, 1.0321818990632892e-06, 1.0395636991233914e-06, 1.1400030643926584e-06, 1.1246974054301973e-06, 1.1164495390403317e-06, 1.1067602372349938e-06, 1.0329455335522653e-06, 1.0215419479209231e-06, 1.0516191650822293e-06, 1.094025606107607e-06, 1.9820049601548817e-06, 2.02492083190009e-06, 2.027213895416935e-06, 2.0755251171067357e-06, 7.982699571584817e-06, 7.091677616699599e-06, 6.196530648594489e-06, 5.415331088443054e-06, 4.509272457653424e-06, 3.817227934632683e-06, 3.0303874609671766e-06, 2.63499441643944e-06, 1.753965875650465e-06, 1.454571702197427e-06, 1.3003293588553788e-06, 1.2596876786119537e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4973254312499193e-05, "first_to_last_ratio": 1.0253056867861459, "imbalance_abs_log_ratio": 0.024990799139217344, "last_quartile_mean": 2.4356886569876224e-05, "mean": 3.072147035254602e-05, "normalized": [ 0.9236862150549096, 1.003433477922723, 0.8748851598635943, 0.8394575367992361, 0.8363452717314914, 0.7383780782804547, 0.6614970076240289, 0.6254576365443, 0.6885640744783226, 0.6487931115557062, 0.6046011264871656, 0.5741697415936372, 0.5386709492382923, 0.50629821766184, 0.5054001346288222, 0.597129298278525, 1.2072440909181623, 1.2203532137980602, 1.1113429384319282, 1.0425775353104243, 3.4234650488600016, 2.717828515395231, 2.0856806102438092, 1.6821053762082474, 1.4314304363976806, 1.2175336783265207, 0.9576282827857749, 0.8451765399742887, 0.5586553690400654, 0.4724074841866198, 0.4327641759989182, 0.42703966638121854 ], "peak_layer": 21, "peak_normalized": 3.4234650488600016, "population_cv": 0.6561028335210565, "rest_mean": 2.350703370110649e-05, "spike_contrast": 2.9641948514771395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.967942827031947e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.837699867086485e-05, 3.082695184275508e-05, 2.68777585006319e-05, 2.5789369828999043e-05, 2.5693756469991058e-05, 2.268406024086289e-05, 2.032216070801951e-05, 1.9214978237869218e-05, 2.1153700799914077e-05, 1.993187834159471e-05, 1.8574235582491383e-05, 1.7639338693697937e-05, 1.654876359680202e-05, 1.555422568344511e-05, 1.5526635252172127e-05, 1.8344690033700317e-05, 3.70883135474287e-05, 3.7491045077331364e-05, 3.414208913454786e-05, 3.20295148412697e-05, 0.00010517388000153005, 8.349568815901875e-05, 6.407517503248528e-05, 5.167675044503994e-05, 4.397564771352336e-05, 3.740442480193451e-05, 2.941974889836274e-05, 2.5965066015487537e-05, 1.7162714357255027e-05, 1.4513052519760095e-05, 1.3295151802594773e-05, 1.311928645009175e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9351784885657253e-05, "first_to_last_ratio": 1.2061546872991604, "imbalance_abs_log_ratio": 0.18743735483861235, "last_quartile_mean": 1.604419821887859e-05, "mean": 2.2264120815407296e-05, "normalized": [ 0.9988983621888606, 1.0764878394047224, 0.9445299593886946, 0.9051803111065253, 0.8729387681656537, 0.7752191091718748, 0.7092152505168287, 0.6710625142970259, 0.7281617557104753, 0.68279470352268, 0.6397189413546165, 0.6085099699564221, 0.6142476722616719, 0.5792470223407198, 0.5731808432472965, 0.6526049864885849, 1.2669803566037365, 1.2796741605033877, 1.1379908185023777, 1.069524915820068, 3.416144871325449, 2.6441962324509487, 1.8911944255720232, 1.497254472533639, 1.262212643271896, 1.084844818314985, 0.8878285176224576, 0.8046030960517679, 0.5007719935996711, 0.43406073132059764, 0.3983201201335122, 0.39239981725082984 ], "peak_layer": 21, "peak_normalized": 3.416144871325449, "population_cv": 0.6261751709955039, "rest_mean": 1.755484478322884e-05, "spike_contrast": 2.716868874553423, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.769421138917096e-05, "top_five_layers": [ 21, 22, 23, 24, 18 ], "top_five_spike_overlap": 4 }, "values": [ 2.2239593818085268e-05, 2.3967055312823504e-05, 2.1029129129601642e-05, 2.015304380620364e-05, 1.9435214198892936e-05, 1.7259571905015036e-05, 1.5790054021636024e-05, 1.494061689299997e-05, 1.6211881302297115e-05, 1.5201823771349154e-05, 1.4242779798223637e-05, 1.3547939488489646e-05, 1.3675684385816567e-05, 1.2896425687358715e-05, 1.2761367543134838e-05, 1.45296762639191e-05, 2.820820373017341e-05, 2.849082011380233e-05, 2.5336365069961175e-05, 2.381203194090631e-05, 7.605746213812381e-05, 5.8870704378932714e-05, 4.2105781176360324e-05, 3.333505446789786e-05, 2.8102054784540087e-05, 2.4153116100933403e-05, 1.976672137971036e-05, 1.791378053894732e-05, 1.1149248166475445e-05, 9.663980563345831e-06, 8.868247277860064e-06, 8.736436939216219e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.32552014394605e-05, "first_to_last_ratio": 0.8017636202998193, "imbalance_abs_log_ratio": 0.22094145234011275, "last_quartile_mean": 7.889507560321363e-05, "mean": 8.741289354929904e-05, "normalized": [ 0.8167213061822914, 0.8928199264535667, 0.7770023518116121, 0.7441008131359753, 0.7576510760758486, 0.6634661276454551, 0.5770587298246667, 0.5602754300157883, 0.6301536495616524, 0.6007155884581821, 0.569798578315965, 0.5464900651195925, 0.44900852862753954, 0.42205691313896365, 0.42493631904268764, 0.5138066897392368, 1.1202499594926825, 1.1323452455529242, 1.0885892297612487, 0.9948773152900628, 3.437247368841268, 2.8322050553727807, 2.3105289217473306, 1.9174427790824178, 1.6051556310819608, 1.3857412887469938, 1.1028008079933802, 0.9896415626766426, 0.6559632164073802, 0.5273861687480318, 0.4828880564820797, 0.470875299573792 ], "peak_layer": 21, "peak_normalized": 3.437247368841268, "population_cv": 0.7050518993663308, "rest_mean": 6.441818806234989e-05, "spike_contrast": 3.284542914712745, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00021158430317882448, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 7.139197259675711e-05, 7.804397318977863e-05, 6.792002386646345e-05, 6.504400516860187e-05, 6.622847286053002e-05, 5.799549398943782e-05, 5.04423733218573e-05, 4.897529652225785e-05, 5.508355388883501e-05, 5.25102877872996e-05, 4.980774247087538e-05, 4.777027788804844e-05, 3.9249134715646505e-05, 3.689321601996198e-05, 3.714491322170943e-05, 4.491332947509363e-05, 9.792429045774043e-05, 9.898157441057265e-05, 9.515673446003348e-05, 8.696510485606268e-05, 0.00030045973835512996, 0.0002475712390150875, 0.0002019700186792761, 0.0001676092215348035, 0.00014031129830982536, 0.00012113165576010942, 9.639900963520631e-05, 8.650743257021531e-05, 5.733964280807413e-05, 4.610035102814436e-05, 4.2210642277495936e-05, 4.1160572436638176e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "36b6c956f67457ef008d5eacc14cf9477822b0791655ecd816dc4fb89c68ffd5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059407927095890045, 0.07886742800474167, 0.16177938878536224, 0.15622827410697937, 0.1895449161529541, 0.15217198431491852, 0.14916078746318817, 0.14915435016155243, 0.16036193072795868, 0.12247956544160843, 0.12010690569877625, 0.12369577586650848, 0.14881008863449097, 0.10321364551782608, 0.1168464794754982, 0.16267083585262299, 0.21974295377731323, 0.1438056081533432, 0.14797262847423553, 0.15148812532424927, 0.1692686378955841, 0.07994630187749863, 0.09600856900215149, 0.09550122171640396, 0.10489360988140106, 0.07298607379198074, 0.08083610236644745, 0.0843365490436554, 0.09205732494592667, 0.07988084107637405, 0.08835897594690323, 0.09466564655303955 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.231516780668244e-06, "first_to_last_ratio": 1.2181687678086963, "imbalance_abs_log_ratio": 0.1973487211082494, "last_quartile_mean": 4.29457470829675e-06, "mean": 7.48252770677027e-06, "normalized": [ 0.7049335753209298, 0.8672771403140204, 0.8924172675590143, 0.5058818096791461, 0.5579230679041902, 0.8745502145818771, 0.7289352734533796, 0.46139662247417734, 0.4048700191822373, 0.5891204135113416, 0.5910949790604852, 0.5127332908040538, 0.2632672019864895, 0.3082220550136737, 0.22207938427345525, 1.138921629601989, 0.7844735000286154, 1.3148028500337634, 2.424187191214654, 2.3081308703788386, 3.6462484228858485, 3.003416179964025, 2.691692507346097, 1.611848128667073, 0.9027728276579748, 0.8071999655073968, 0.8661618055353845, 0.5565823202716984, 0.6137168515164881, 0.45586796048570105, 0.23946068136284482, 0.14981399242313648 ], "peak_layer": 21, "peak_normalized": 3.6462484228858485, "population_cv": 0.8597567898445678, "rest_mean": 5.5825260090016485e-06, "spike_contrast": 3.178227355521773, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7742536874720828e-05, "top_five_layers": [ 21, 22, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 5.274685008771485e-06, 6.489425231848145e-06, 6.6775369305105414e-06, 3.785274657275295e-06, 4.174674813839374e-06, 6.543846211570781e-06, 5.454278380057076e-06, 3.4524130114732543e-06, 3.0294511361717014e-06, 4.408109816722572e-06, 4.422884558152873e-06, 3.836541054624831e-06, 1.969904133147793e-06, 2.306280066477484e-06, 1.6617151459286106e-06, 8.52201264933683e-06, 5.869844699191162e-06, 9.838048754318152e-06, 1.8139047824661247e-05, 1.7270653188461438e-05, 2.728315485001076e-05, 2.2473144781542942e-05, 2.0140663764323108e-05, 1.2060698281857185e-05, 6.755022695870139e-06, 6.039896106813103e-06, 6.4810797084646765e-06, 4.164642632531468e-06, 4.592153345583938e-06, 3.4110446449631127e-06, 1.7917711829795735e-06, 1.12098734916799e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.9753211972783902e-05, "first_to_last_ratio": 1.1556699430673998, "imbalance_abs_log_ratio": 0.14468021310494286, "last_quartile_mean": 1.709243377945313e-05, "mean": 2.9343291515715464e-05, "normalized": [ 0.6423992673864061, 0.7671882759686803, 0.8850259786401377, 0.5070019793112642, 0.5368844244513357, 0.8343894815896736, 0.7353960442078943, 0.4771259191368489, 0.4241382669659466, 0.5855916700343277, 0.5778160836542935, 0.5253742530620523, 0.25966342686064026, 0.31295329557343554, 0.2348431281691379, 1.1373249210235588, 0.8069155942709877, 1.3369118805785314, 2.3460920286428437, 2.4540482840769475, 3.5680018751807268, 3.0328373621092837, 2.734987268865684, 1.6170984312591716, 0.9302523641112638, 0.8177707797658776, 0.8816015913798516, 0.5588721385124307, 0.6102613176308225, 0.4675196143273077, 0.24642719158815024, 0.14728586166448673 ], "peak_layer": 21, "peak_normalized": 3.5680018751807268, "population_cv": 0.8610884962987538, "rest_mean": 2.1862733067084468e-05, "spike_contrast": 3.1898256692946436, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.973830713832285e-05, "top_five_layers": [ 21, 22, 23, 20, 19 ], "top_five_spike_overlap": 3 }, "values": [ 1.8850108972401358e-05, 2.251182922918815e-05, 2.5969575290218927e-05, 1.4877106877975166e-05, 1.5753956176922657e-05, 2.4483733795932494e-05, 2.157894050469622e-05, 1.400044493493624e-05, 1.244561281055212e-05, 1.7183187082991935e-05, 1.6955025785136968e-05, 1.541620986245107e-05, 7.619379630341427e-06, 9.183079782815184e-06, 6.891070370329544e-06, 3.337285670568235e-05, 2.3677559511270374e-05, 3.9229395042639226e-05, 6.884206231916323e-05, 7.200985419331118e-05, 0.00010469691915204749, 8.899343083612621e-05, 8.025352872209623e-05, 4.7450990678044036e-05, 2.72966663033003e-05, 2.3996086383704096e-05, 2.586909249657765e-05, 1.6399148080381565e-05, 1.790707574400585e-05, 1.3718564332521055e-05, 7.230984920170158e-06, 4.321851974964375e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.017517346306703985, "first_to_last_ratio": 1.1556699645631487, "imbalance_abs_log_ratio": 0.1446802317051919, "last_quartile_mean": 0.015157741261646152, "mean": 0.026021924597444013, "normalized": [ 0.642399269560858, 0.7671882665980466, 0.8850260023995508, 0.5070019973604465, 0.5368843965740409, 0.8343894944034302, 0.7353960815145425, 0.47712593296252226, 0.4241382778148869, 0.5855916847742128, 0.5778160385433538, 0.5253742512409212, 0.25966343220529986, 0.31295327953435204, 0.23484312335084737, 1.1373249608588862, 0.8069155777916533, 1.3369118107572777, 2.346092040189918, 2.45404832250407, 3.5680018220110936, 3.0328371668368583, 2.73498744787703, 1.6170984888723343, 0.9302523630738624, 0.8177707318164164, 0.8816015513659362, 0.5588721480759673, 0.6102613538594246, 0.4675196323727517, 0.24642719276174477, 0.14728586013746342 ], "peak_layer": 21, "peak_normalized": 3.5680018220110936, "population_cv": 0.861088491107969, "rest_mean": 0.01938809050867955, "spike_contrast": 3.189825663805616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.061844628676772116, "top_five_layers": [ 21, 22, 23, 20, 19 ], "top_five_spike_overlap": 3 }, "values": [ 0.01671646535396576, 0.019963715225458145, 0.023030079901218414, 0.013193167746067047, 0.01397076528519392, 0.021712420508265495, 0.019136421382427216, 0.012415735051035881, 0.011036894284188747, 0.015238222666084766, 0.015035885386168957, 0.013671249151229858, 0.006756942253559828, 0.008143646642565727, 0.0061110700480639935, 0.029595384374260902, 0.02099749632179737, 0.03478901833295822, 0.06104983016848564, 0.06385906040668488, 0.09284627437591553, 0.07892026007175446, 0.07116963714361191, 0.04208001494407654, 0.02420695684850216, 0.021279968321323395, 0.022940969094634056, 0.014542928896844387, 0.015880174934864044, 0.01216576062142849, 0.0064125098288059235, 0.003832661546766758 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.912403284620723e-06, "first_to_last_ratio": 0.7440148464151746, "imbalance_abs_log_ratio": 0.29569428948903514, "last_quartile_mean": 2.570383230704465e-06, "mean": 2.2519678122279174e-06, "normalized": [ 3.7363448723044836, 1.2999770872722904, 0.2528355179896465, 0.3447212899074926, 0.28850832739316085, 0.26077899448522845, 0.3004387502401656, 0.310109614891652, 0.26155308064966604, 0.34461906116108865, 0.3430527653161843, 0.4158029818571386, 0.21524181782036755, 0.3598800713242707, 0.3675641897039758, 0.3564396575202899, 0.3380877296575, 0.6093802060812419, 0.9402690126586363, 1.079633600656378, 1.9776909756331749, 3.4168782728753038, 2.651381140535597, 2.3976565605938904, 1.2926094488981195, 1.6689818693255922, 1.661631597217477, 1.2056041530585988, 1.4143405120828008, 0.8908828858291652, 0.5365608273760537, 0.460543127683369 ], "peak_layer": 1, "peak_normalized": 3.7363448723044836, "population_cv": 0.9346042008561078, "rest_mean": 1.6901254972018016e-06, "spike_contrast": 3.127528885943908, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.285916313368944e-06, "top_five_layers": [ 1, 22, 23, 24, 21 ], "top_five_spike_overlap": 4 }, "values": [ 8.414128387812525e-06, 2.927506557171e-06, 5.693774483006564e-07, 7.763012490613619e-07, 6.497114668491122e-07, 5.872659016858961e-07, 6.765783950868354e-07, 6.983568709983956e-07, 5.890091188121005e-07, 7.760710332149756e-07, 7.725437853878248e-07, 9.363749313706649e-07, 4.847176455768931e-07, 8.104383368845447e-07, 8.277427241409896e-07, 8.026906357372354e-07, 7.613626848979038e-07, 1.3723046095037716e-06, 2.117455551342573e-06, 2.4313001176778926e-06, 4.453696419659536e-06, 7.694699888816103e-06, 5.970824986434309e-06, 5.399445399234537e-06, 2.910914872700232e-06, 3.758493448913214e-06, 3.741940872714622e-06, 2.714981746976264e-06, 3.1850493087404175e-06, 2.0062395833519986e-06, 1.208317712553253e-06, 1.037128299685719e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.3613449709737324e-05, "first_to_last_ratio": 1.2961798669265885, "imbalance_abs_log_ratio": 0.25942137450895547, "last_quartile_mean": 1.0502747386453848e-05, "mean": 1.94846966365958e-05, "normalized": [ 0.6224917639630821, 0.8091508021640391, 0.9260582357582028, 0.5438708025741, 0.539202130448075, 0.86685282634047, 0.7739497051591201, 0.5078152293450356, 0.44530646133543333, 0.6024460206063752, 0.6204465476276101, 0.5023513612748098, 0.2919364277081767, 0.34034090010393764, 0.2666101183752142, 1.157137994849184, 0.8274903470382167, 1.369617562543775, 2.446429377746549, 2.1498756230367086, 3.341616124095529, 3.2302364132353425, 2.8911802848363917, 1.6153833341689192, 0.9062162391904391, 0.7992110395710725, 0.8759188816263488, 0.4509435491843002, 0.543934283818765, 0.3834922528262834, 0.21307834519941646, 0.13940901424907648 ], "peak_layer": 21, "peak_normalized": 3.341616124095529, "population_cv": 0.8584035053051066, "rest_mean": 1.4444198734930035e-05, "spike_contrast": 3.2333662920774784, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.670338530559093e-05, "top_five_layers": [ 21, 22, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.2129063179600053e-05, 1.5766057913424447e-05, 1.8043963791569695e-05, 1.0597157597658224e-05, 1.0506189937586896e-05, 1.689036434981972e-05, 1.5080175217008218e-05, 9.89462569123134e-06, 8.676661309436895e-06, 1.1738477951439563e-05, 1.208921275974717e-05, 9.788163879420608e-06, 5.688292731065303e-06, 6.6314391915511806e-06, 5.194817276787944e-06, 2.2546482796315104e-05, 1.612339838175103e-05, 2.668658271431923e-05, 4.7667934268247336e-05, 4.1889674321282655e-05, 6.511037645395845e-05, 6.294017657637596e-05, 5.633377077174373e-05, 3.147525421809405e-05, 1.7657348507782444e-05, 1.557238465466071e-05, 1.7067013686755672e-05, 8.786498256085906e-06, 1.0598394510452636e-05, 7.472230208804831e-06, 4.1517669160384685e-06, 2.716342351050116e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.224245218319993e-06, "first_to_last_ratio": 1.42352192746992, "imbalance_abs_log_ratio": 0.3531340315301507, "last_quartile_mean": 6.47987575064235e-06, "mean": 1.2729493800378577e-05, "normalized": [ 0.575325501428242, 0.8765327600629662, 0.9593474390198574, 0.5853400578880529, 0.5587901873677211, 0.889540337545407, 0.8146648699468889, 0.5375441750172566, 0.47404798886848176, 0.6359131178887726, 0.6792320083196889, 0.46547624987616953, 0.32872806943308774, 0.37175338073287834, 0.3016769923893083, 1.1851024317290795, 0.8670161235264493, 1.4111720806791088, 2.506650783858839, 1.9418762691937648, 2.8977084750056923, 3.415371148318928, 3.0397221818581524, 1.609113290765392, 0.8688260399415566, 0.8080203647056072, 0.8707912839378436, 0.3953034784106991, 0.49891926408419757, 0.31272264819953055, 0.18262343848706983, 0.13514756151330948 ], "peak_layer": 22, "peak_normalized": 3.415371148318928, "population_cv": 0.8455005471645312, "rest_mean": 9.509053913663794e-06, "spike_contrast": 3.16749378666982, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0119869188638403e-05, "top_five_layers": [ 22, 23, 21, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 7.3236024036305025e-06, 1.1157818335050251e-05, 1.221200727741234e-05, 7.451082637999207e-06, 7.11311622580979e-06, 1.1323398211970925e-05, 1.0370271411375143e-05, 6.842665243311785e-06, 6.0343909353832714e-06, 8.094852091744542e-06, 8.64627963892417e-06, 5.92527703702217e-06, 4.184541921858909e-06, 4.732232355308952e-06, 3.840195404336555e-06, 1.5085754057508893e-05, 1.1036676369258203e-05, 1.7963506252272055e-05, 3.19083956128452e-05, 2.471910192980431e-05, 3.688636206788942e-05, 4.347594585851766e-05, 3.8694124668836594e-05, 2.0483197658904828e-05, 1.1059715689043514e-05, 1.0285690223099664e-05, 1.108473225031048e-05, 5.032013177697081e-06, 6.350989679049235e-06, 3.980801011493895e-06, 2.3247039280249737e-06, 1.7203600464199553e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.924221596207644e-05, "first_to_last_ratio": 1.220088035622856, "imbalance_abs_log_ratio": 0.19892301648833594, "last_quartile_mean": 3.216342986434029e-05, "mean": 5.811260362520443e-05, "normalized": [ 0.6785990759991457, 0.7772257654622007, 0.8732309863789576, 0.5020979620112224, 0.5168098626008664, 0.8345517579225961, 0.7379646492413534, 0.4817512656456255, 0.4163654546730875, 0.5357163632735232, 0.5807450859096301, 0.520383542426531, 0.2612071324961203, 0.31411274991679294, 0.24095333784569256, 1.12561496002337, 0.8021979217859124, 1.3082348163567306, 2.4237584214732073, 2.344022957737727, 3.760809922544462, 3.1425833141182378, 2.8316122260106797, 1.5617115119945912, 0.9609041629692773, 0.7679731552856547, 0.8714908320820871, 0.46503806554022425, 0.5652008886197345, 0.42614135692315014, 0.23070316107128952, 0.14028733366031948 ], "peak_layer": 21, "peak_normalized": 3.760809922544462, "population_cv": 0.8951816421708676, "rest_mean": 4.249189027581856e-05, "spike_contrast": 3.352744601078911, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014246445571188814, "top_five_layers": [ 21, 22, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 3.943515912396833e-05, 4.516661283560097e-05, 5.0745726184686646e-05, 2.9178219847381115e-05, 3.003316669492051e-05, 4.849797551287338e-05, 4.288504715077579e-05, 2.79958203464048e-05, 2.4196080630645156e-05, 3.113187267445028e-05, 3.374860898475163e-05, 3.0240842534112744e-05, 1.5179426554823294e-05, 1.8253909729537554e-05, 1.4002425814396702e-05, 6.541241600643843e-05, 4.661780985770747e-05, 7.60249313316308e-05, 0.00014085091243032366, 0.00013621727703139186, 0.00021855045633856207, 0.00018262369849253446, 0.0001645523589104414, 9.075512207346037e-05, 5.584064274444245e-05, 4.462891956791282e-05, 5.064460128778592e-05, 2.7024572773370892e-05, 3.284529520897195e-05, 2.476418376318179e-05, 1.340676135441754e-05, 8.152462214638945e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c1e7ba7b2fabbdfa228a4a3c33df9fd2cb9236dae2c5351a1ecf233f5b8dce87", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05594591051340103, 0.07073435187339783, 0.06750936061143875, 0.10916222631931305, 0.11270010471343994, 0.14727257192134857, 0.11801524460315704, 0.13372685015201569, 0.09399056434631348, 0.10827244818210602, 0.10214843600988388, 0.09725081920623779, 0.0902416780591011, 0.09460548311471939, 0.11164536327123642, 0.19675607979297638, 0.15372900664806366, 0.10524243861436844, 0.10666082799434662, 0.11902136355638504, 0.05537491664290428, 0.06347332894802094, 0.07961282879114151, 0.09120825678110123, 0.07538540661334991, 0.07336290925741196, 0.07823792845010757, 0.08682125806808472, 0.057621169835329056, 0.0771951824426651, 0.09140052646398544, 0.09293081611394882 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.0007654601149625e-06, "first_to_last_ratio": 0.21469374652673343, "imbalance_abs_log_ratio": 1.5385427010854755, "last_quartile_mean": 1.397695791638398e-05, "mean": 9.062807375670445e-06, "normalized": [ 0.6788277855296613, 0.33022065578659243, 0.5185396106172449, 0.3147344808808441, 0.1493975773164771, 0.16583194042116398, 0.21736502256384724, 0.2739446631895577, 0.2986054592892529, 0.2841589081967956, 0.3703811746673963, 0.4629251306676604, 0.33656653078263976, 0.3194237016069162, 0.33703323000014157, 0.2868358429246865, 0.3379218451804822, 0.44189350931307353, 0.5391000167958766, 0.7544514236626646, 3.62784685261383, 3.4055741874667143, 2.8075043335903698, 2.4030544923752126, 2.128695565263529, 2.0503888456814066, 2.0115823114995672, 1.62589115957208, 1.6316185991465908, 1.0929664148973441, 0.9374347986692717, 0.8592839298311081 ], "peak_layer": 21, "peak_normalized": 3.62784685261383, "population_cv": 0.9767141058146375, "rest_mean": 5.916779522757915e-06, "spike_contrast": 4.402962402299404, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.605135778139811e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.15208546150825e-06, 2.9927261948614614e-06, 4.699424607679248e-06, 2.852377974704723e-06, 1.3539614656110643e-06, 1.5029029327706667e-06, 1.9699373297044076e-06, 2.4827077140798792e-06, 2.7062037588621024e-06, 2.5752774490683805e-06, 3.3566932415851625e-06, 4.195401288598077e-06, 3.0502376375807216e-06, 2.8948754788871156e-06, 3.0544672426913166e-06, 2.599537992864498e-06, 3.0625205909018405e-06, 4.0047957554634195e-06, 4.885759608441731e-06, 6.837447926955065e-06, 3.287847721367143e-05, 3.086406286456622e-05, 2.5443870981689543e-05, 2.1778419977636077e-05, 1.929195786942728e-05, 1.8582279153633863e-05, 1.8230583009426482e-05, 1.4735138393007219e-05, 1.4787045074626803e-05, 9.905344086291734e-06, 8.495791007590014e-06, 7.787524737068452e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.1920566521439468e-05, "first_to_last_ratio": 0.21385698745540163, "imbalance_abs_log_ratio": 1.5424477702795347, "last_quartile_mean": 5.574083252213313e-05, "mean": 3.626819514579438e-05, "normalized": [ 0.6714239045350167, 0.3268984674053377, 0.5163313168674706, 0.31628734338757886, 0.14365858953035543, 0.1607028969713726, 0.2191876706792618, 0.2749357246554526, 0.3063367204191196, 0.290288819397969, 0.37595208658458074, 0.4631048978627428, 0.3453629561715752, 0.32617163789727366, 0.3475938493090438, 0.3029470728685741, 0.34742911900824086, 0.44840259940070093, 0.5403498924783475, 0.7515227531851133, 3.6289752746901156, 3.3976055612895637, 2.7998333290588935, 2.4034438268675102, 2.124889978824914, 2.0413065915513173, 2.0135506517890525, 1.6239375357758048, 1.6208285992904423, 1.0902980270168148, 0.928318564252677, 0.8521237409777672 ], "peak_layer": 21, "peak_normalized": 3.6289752746901156, "population_cv": 0.9738982135151035, "rest_mean": 2.3702275703490834e-05, "spike_contrast": 4.3930026566562255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010412416013423353, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.4351333195227198e-05, 1.185601740871789e-05, 1.8726404960034415e-05, 1.1471171092125587e-05, 5.210237759456504e-06, 5.82840402785223e-06, 7.94954121374758e-06, 9.971422514354344e-06, 1.1110279956483282e-05, 1.0528251550567802e-05, 1.363510364171816e-05, 1.679597880865913e-05, 1.2525691090559121e-05, 1.1829656614281703e-05, 1.2606601558218244e-05, 1.0987343557644635e-05, 1.2600627087522298e-05, 1.6262752978946082e-05, 1.959751534741372e-05, 2.7256373869022354e-05, 0.00013161638344172388, 0.00012322502152528614, 0.00010154490155400708, 8.71685697347857e-05, 7.706592441536486e-05, 7.403450581477955e-05, 7.302784797502682e-05, 5.8897283452097327e-05, 5.8784527936950326e-05, 3.954314161092043e-05, 3.366843884577975e-05, 3.0904990126146004e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.010571277292910963, "first_to_last_ratio": 0.2138569831347789, "imbalance_abs_log_ratio": 1.5424477904828622, "last_quartile_mean": 0.04943152726627886, "mean": 0.032162998293642886, "normalized": [ 0.6714238750017825, 0.3268984397015616, 0.5163313088425289, 0.31628733258408726, 0.14365859237299086, 0.16070289962904324, 0.21918766128478195, 0.2749357162222242, 0.3063367194831541, 0.29028882869370426, 0.3759520938745508, 0.46310490191839176, 0.345362968353024, 0.3261716356763452, 0.3475938513217391, 0.302947090830093, 0.34742911872813526, 0.4484025830910049, 0.5403498943254139, 0.7515227169830379, 3.6289754691077154, 3.39760554441039, 2.7998334990791727, 2.4034437339282073, 2.12489003938986, 2.0413065340618877, 2.013550612247264, 1.6239375562448826, 1.6208285723970963, 1.0902980031467058, 0.9283185200253238, 0.8521236870439002 ], "peak_layer": 21, "peak_normalized": 3.6289754691077154, "population_cv": 0.9738982344432222, "rest_mean": 0.021019414843370516, "spike_contrast": 4.393002831581538, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09233834892511368, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.021595004945993423, 0.01051403395831585, 0.016606763005256653, 0.010172748938202858, 0.004620491061359644, 0.0051686870865523815, 0.0070497323758900166, 0.00884275697171688, 0.009852707386016846, 0.009336559101939201, 0.012091746553778648, 0.01489484217017889, 0.011107908561825752, 0.010490657761693, 0.011179660446941853, 0.009743686765432358, 0.011174362152814865, 0.014421971514821053, 0.0173792727291584, 0.024171223863959312, 0.11671873182058334, 0.10927718132734299, 0.09005104005336761, 0.07730195671319962, 0.06834283471107483, 0.06565453857183456, 0.06476182490587234, 0.052230700850486755, 0.05213070660829544, 0.035067252814769745, 0.029857506975531578, 0.027406852692365646 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.322070665338742e-07, "first_to_last_ratio": 0.19582650195201706, "imbalance_abs_log_ratio": 1.630526205889656, "last_quartile_mean": 3.7390601335118845e-06, "mean": 2.320639892694487e-06, "normalized": [ 0.6549870344311939, 0.3169062144097135, 0.5191630414175221, 0.3243434965301332, 0.105717119181206, 0.15182526527542842, 0.19569754579298845, 0.2555160344471036, 0.23140272407874934, 0.23882207606587388, 0.31847255362240917, 0.41661698388421226, 0.3042701489276803, 0.32496723001951616, 0.2926718756416169, 0.14809689981232488, 0.29560203174175614, 0.3094570768459675, 0.46180841517164245, 0.6023706663610227, 3.6555712774602895, 3.5328017931896913, 2.9040888216519436, 2.54906806729493, 2.239269068685756, 2.2005088234873713, 2.152636739293442, 1.694221234427147, 1.6655655501398623, 1.1343787866859985, 0.9462733663708415, 0.8569020376546662 ], "peak_layer": 21, "peak_normalized": 3.6555712774602895, "population_cv": 1.0356066456325392, "rest_mean": 1.471388915037837e-06, "spike_contrast": 4.6939290499295305, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.9065951720403975e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.519989041298686e-06, 7.354252034019737e-07, 1.204790464726102e-06, 7.526844569838431e-07, 2.453313641126442e-07, 3.5233176731708227e-07, 4.541435316696152e-07, 5.929607027610473e-07, 5.370023927753209e-07, 5.542200369745842e-07, 7.390601126644469e-07, 9.668179927757592e-07, 7.061014457576675e-07, 7.541319178017147e-07, 6.791860300836561e-07, 3.436795736888598e-07, 6.859858672214614e-07, 7.181384376053757e-07, 1.0716910310293315e-06, 1.3978853985463502e-06, 8.483264537062496e-06, 8.198360774258617e-06, 6.739344371453626e-06, 5.91546904615825e-06, 5.196537131268997e-06, 5.106588560011005e-06, 4.995494691684144e-06, 3.931677383661736e-06, 3.865177859552205e-06, 2.6324846658098977e-06, 2.1959597233944805e-06, 1.988561052712612e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 8.062098231675918e-06, "first_to_last_ratio": 0.23172405564077758, "imbalance_abs_log_ratio": 1.4622080305731233, "last_quartile_mean": 3.479180531940074e-05, "mean": 2.387608699905286e-05, "normalized": [ 0.7258751481874687, 0.3420421656378502, 0.5451754824595273, 0.28689678416588577, 0.14555997121980288, 0.15910389252688525, 0.22744868868375562, 0.26921093638205285, 0.331277324094237, 0.3124442700204022, 0.40511967495320234, 0.5118199353326568, 0.40767330553010017, 0.38094082583923083, 0.3799217810408602, 0.2343632382817309, 0.37127597519882294, 0.47558084035172465, 0.5644703292999084, 0.7840592083426426, 3.8152513101159804, 3.5348515650624037, 2.7948696249240457, 2.3373113491061237, 2.169069307958383, 1.9545846045943898, 1.8833096737193575, 1.503620407983867, 1.520809170499446, 0.9931364071654021, 0.8480889104968432, 0.7848378908250101 ], "peak_layer": 21, "peak_normalized": 3.8152513101159804, "population_cv": 0.9831166322047221, "rest_mean": 1.5341400049456518e-05, "spike_contrast": 4.560431010294372, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.996339652687311e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.733105818857439e-05, 8.16662850411376e-06, 1.3016657248954289e-05, 6.84997257849318e-06, 3.475402536423644e-06, 3.7987783798598684e-06, 5.4305846788338386e-06, 6.427703738154378e-06, 7.909606210887432e-06, 7.459946573362686e-06, 9.672672604210675e-06, 1.2220257303852122e-05, 9.733643310028128e-06, 9.09537629922852e-06, 9.07104549696669e-06, 5.595677066594362e-06, 8.864617484505288e-06, 1.1355009519320447e-05, 1.3477342690748628e-05, 1.8720265870797448e-05, 9.109327220357955e-05, 8.439842349616811e-05, 6.673055031569675e-05, 5.580584911513142e-05, 5.178888750378974e-05, 4.666783206630498e-05, 4.4966065615881234e-05, 3.5900571674574167e-05, 3.631097206380218e-05, 2.3712211259407923e-05, 2.024904460995458e-05, 1.873885776149109e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.398108925191991e-06, "first_to_last_ratio": 0.2553243719405187, "imbalance_abs_log_ratio": 1.3652204953642635, "last_quartile_mean": 2.114216079007747e-05, "mean": 1.5431339967619806e-05, "normalized": [ 0.775450361876563, 0.35993336560900263, 0.5716811811488363, 0.2647889720822481, 0.15443931558124985, 0.1609106807603714, 0.24114708063049767, 0.27016631249985124, 0.3542374939344869, 0.3341618028764158, 0.4390379760950059, 0.5611696202016426, 0.4905651293202641, 0.44854876717590725, 0.427299912149135, 0.17007282318732395, 0.4083388005340233, 0.5099761158699698, 0.6066217231343173, 0.8107855537457127, 4.065115409247974, 3.706799247664954, 2.7077946969814244, 2.2003225686125982, 2.1672422326648038, 1.812476034013852, 1.6953296147818004, 1.3306288007797171, 1.4218486794014777, 0.9438307759525454, 0.8255782054336522, 0.7637007460523768 ], "peak_layer": 21, "peak_normalized": 4.065115409247974, "population_cv": 0.9906146212599648, "rest_mean": 9.80331644010827e-06, "spike_contrast": 4.674200541839902, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.58226670161821e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.1966238162131049e-05, 5.554254130402114e-06, 8.821806659398135e-06, 4.08604864787776e-06, 2.3832055831007892e-06, 2.4830674192344304e-06, 3.7212225834082346e-06, 4.169028215983417e-06, 5.466359198180726e-06, 5.156564384378726e-06, 6.774944267817773e-06, 8.659599188831635e-06, 7.57007728680037e-06, 6.921708518348169e-06, 6.593810212507378e-06, 2.6244515538564883e-06, 6.301214853010606e-06, 7.869614819355775e-06, 9.360986041428987e-06, 1.2511507520684972e-05, 6.27301778877154e-05, 5.720087938243523e-05, 4.1784900531638414e-05, 3.395392559468746e-05, 3.344345168443397e-05, 2.7968933864030987e-05, 2.6161207642871886e-05, 2.0533385395538062e-05, 2.1941030354355462e-05, 1.4564573575626127e-05, 1.2739777957904153e-05, 1.1784925845859107e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.3460118995899393e-05, "first_to_last_ratio": 0.19614811350083294, "imbalance_abs_log_ratio": 1.6288852240081404, "last_quartile_mean": 0.00011960410211031558, "mean": 7.52558852070706e-05, "normalized": [ 0.658313079608564, 0.30463294507451294, 0.5014382981404909, 0.28041349059373355, 0.1296990900515029, 0.1558393800657072, 0.20228299557148446, 0.2612847052904527, 0.3002182827700767, 0.28296321216930564, 0.37117751666594523, 0.4617148213254951, 0.32553402471276316, 0.31924765387609977, 0.3311215231692876, 0.2688995234151016, 0.3256293540572768, 0.42908371238624793, 0.5189360670040057, 0.7298065590677655, 3.5899584767580643, 3.2958212345047615, 2.8004713972940176, 2.44112076202934, 2.203847377095527, 2.1149132200283227, 2.164777621743948, 1.7470912286112248, 1.6405526614360855, 1.0907160128127615, 0.9220261935993064, 0.8304675790708215 ], "peak_layer": 21, "peak_normalized": 3.5899584767580643, "population_cv": 0.985057448082149, "rest_mean": 4.9247397631309774e-05, "spike_contrast": 4.379961754142385, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002157017181161791, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 4.9541933549335226e-05, 2.292542194481939e-05, 3.7736183003289625e-05, 2.1102765458635986e-05, 9.760619832377415e-06, 1.1727830496965908e-05, 1.5222985894070007e-05, 1.9663211787701584e-05, 2.2593192625208758e-05, 2.129464701283723e-05, 2.793329258565791e-05, 3.474675759207457e-05, 2.449835119477939e-05, 2.4025264792726375e-05, 2.4918843337218277e-05, 2.023627166636288e-05, 2.4505525288986973e-05, 3.2291074603563175e-05, 3.9052993088262156e-05, 5.492223863257095e-05, 0.00027016550302505493, 0.00024802994448691607, 0.00021075195400044322, 0.00018370870384387672, 0.0001658524852246046, 0.00015915966650936753, 0.00016291225620079786, 0.00013147889694664627, 0.00012346124276518822, 8.208279905375093e-05, 6.938789738342166e-05, 6.249757279874757e-05 ] } } } } } }, "step": 4000 }, { "forward_identity_gate": { "comparisons": { "detached_learned": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_value_backward": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "modes": { "detached_learned": { "bits_per_byte": 1.877417995231997, "forward": { "activation_sha256": { "attention_branch_output": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", "mlp_branch_output": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", "post_attention_state": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", "post_mlp_state": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", "pre_attention_input": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", "pre_mlp_input": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a" }, "logits_sha256": "5f6747d51014b7e09e13dda43380fde8b7ba2972518bba8957bd86ff4ed7f868", "loss_nats": 1.3013269901275635, "mixer_summary_sha256": "9abaf96810a98e69acfbc813a9b903aa15d6279ddc462a3c26f1a1846711526f" }, "loss_nats": 1.3013269901275635, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.2200722098350525, "group": 1, "latest_source_mass": 0.06159520894289017, "layer": 1, "max_source_mass": 0.9384047985076904, "mixer_index": 1, "normalized_entropy": 0.3174970857665056, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9384047985076904, "median": 0.949082612991333, "p05": 0.8598991632461548, "p95": 0.9761404991149902 }, { "label": "current_group_partial", "mean": 0.06159520894289017, "median": 0.05091731622815132, "p05": 0.02385946735739708, "p95": 0.14010077714920044 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5651201605796814, "group": 1, "latest_source_mass": 0.3442652225494385, "layer": 2, "max_source_mass": 0.6557347774505615, "mixer_index": 2, "normalized_entropy": 0.815296053174681, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.6557347774505615, "median": 0.6636689305305481, "p05": 0.3472050428390503, "p95": 0.9190996289253235 }, { "label": "current_group_partial", "mean": 0.3442652225494385, "median": 0.3363310396671295, "p05": 0.08090042322874069, "p95": 0.6527949571609497 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4018177092075348, "group": 1, "latest_source_mass": 0.14357808232307434, "layer": 2, "max_source_mass": 0.856421947479248, "mixer_index": 3, "normalized_entropy": 0.5797004164150741, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.856421947479248, "median": 0.8660546541213989, "p05": 0.7540444135665894, "p95": 0.9155591726303101 }, { "label": "current_group_partial", "mean": 0.14357808232307434, "median": 0.13394534587860107, "p05": 0.08444082736968994, "p95": 0.24595558643341064 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.687358021736145, "group": 1, "latest_source_mass": 0.5312775373458862, "layer": 3, "max_source_mass": 0.5312775373458862, "mixer_index": 4, "normalized_entropy": 0.9916480092739848, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.46872249245643616, "median": 0.4674533009529114, "p05": 0.3992259204387665, "p95": 0.5431661605834961 }, { "label": "current_group_partial", "mean": 0.5312775373458862, "median": 0.5325466990470886, "p05": 0.45683377981185913, "p95": 0.6007740497589111 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.2678314447402954, "group": 1, "latest_source_mass": 0.07946884632110596, "layer": 3, "max_source_mass": 0.920531153678894, "mixer_index": 5, "normalized_entropy": 0.38639909712095066, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.920531153678894, "median": 0.9314625263214111, "p05": 0.8444089889526367, "p95": 0.9650533199310303 }, { "label": "current_group_partial", "mean": 0.07946884632110596, "median": 0.06853749603033066, "p05": 0.03494669869542122, "p95": 0.1555909812450409 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.657238245010376, "group": 1, "latest_source_mass": 0.3777551054954529, "layer": 4, "max_source_mass": 0.6222448348999023, "mixer_index": 6, "normalized_entropy": 0.948194356759035, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.6222448348999023, "median": 0.6208515167236328, "p05": 0.5314525961875916, "p95": 0.7065955400466919 }, { "label": "current_group_partial", "mean": 0.3777551054954529, "median": 0.3791484832763672, "p05": 0.2934044897556305, "p95": 0.4685474634170532 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5044373869895935, "group": 1, "latest_source_mass": 0.2207653522491455, "layer": 4, "max_source_mass": 0.7792346477508545, "mixer_index": 7, "normalized_entropy": 0.7277493166488734, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7792346477508545, "median": 0.7866228222846985, "p05": 0.6320486664772034, "p95": 0.9385759830474854 }, { "label": "current_group_partial", "mean": 0.2207653522491455, "median": 0.21337716281414032, "p05": 0.06142403930425644, "p95": 0.367951363325119 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.650418758392334, "group": 2, "latest_source_mass": 0.4602207541465759, "layer": 5, "max_source_mass": 0.5397792458534241, "mixer_index": 8, "normalized_entropy": 0.9383559172337771, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5397792458534241, "median": 0.5213949680328369, "p05": 0.34510529041290283, "p95": 0.7807042598724365 }, { "label": "completed_group_1", "mean": 0.4602207541465759, "median": 0.4786049723625183, "p05": 0.2192957103252411, "p95": 0.6548947095870972 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0032464265823364, "group": 2, "latest_source_mass": 0.37668097019195557, "layer": 5, "max_source_mass": 0.4337552785873413, "mixer_index": 9, "normalized_entropy": 0.913194251448444, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4337552785873413, "median": 0.42766237258911133, "p05": 0.2500072717666626, "p95": 0.6592159271240234 }, { "label": "completed_group_1", "mean": 0.1895637810230255, "median": 0.18978442251682281, "p05": 0.1284591555595398, "p95": 0.25696656107902527 }, { "label": "current_group_partial", "mean": 0.37668097019195557, "median": 0.3803442120552063, "p05": 0.1759825050830841, "p95": 0.568291425704956 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0385558605194092, "group": 2, "latest_source_mass": 0.4949595034122467, "layer": 6, "max_source_mass": 0.4949595034122467, "mixer_index": 10, "normalized_entropy": 0.9453342832879565, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.24478834867477417, "median": 0.23865866661071777, "p05": 0.19890648126602173, "p95": 0.32154381275177 }, { "label": "completed_group_1", "mean": 0.26025211811065674, "median": 0.26050686836242676, "p05": 0.23401768505573273, "p95": 0.2860541343688965 }, { "label": "current_group_partial", "mean": 0.4949595034122467, "median": 0.5017017722129822, "p05": 0.42678141593933105, "p95": 0.5358227491378784 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.031773328781128, "group": 2, "latest_source_mass": 0.33116668462753296, "layer": 6, "max_source_mass": 0.40370824933052063, "mixer_index": 11, "normalized_entropy": 0.9391605568439314, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.40370824933052063, "median": 0.38690465688705444, "p05": 0.18056227266788483, "p95": 0.7100913524627686 }, { "label": "completed_group_1", "mean": 0.2651250660419464, "median": 0.2730395793914795, "p05": 0.12547528743743896, "p95": 0.35344114899635315 }, { "label": "current_group_partial", "mean": 0.33116668462753296, "median": 0.33928292989730835, "p05": 0.1502782255411148, "p95": 0.47667479515075684 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0838522911071777, "group": 2, "latest_source_mass": 0.38127952814102173, "layer": 7, "max_source_mass": 0.38127952814102173, "mixer_index": 12, "normalized_entropy": 0.9865648712351232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35612422227859497, "median": 0.36401283740997314, "p05": 0.2946957051753998, "p95": 0.39828816056251526 }, { "label": "completed_group_1", "mean": 0.2625962495803833, "median": 0.2590169608592987, "p05": 0.24180665612220764, "p95": 0.2929879426956177 }, { "label": "current_group_partial", "mean": 0.38127952814102173, "median": 0.37846946716308594, "p05": 0.3489896059036255, "p95": 0.41648316383361816 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.96053147315979, "group": 2, "latest_source_mass": 0.3146248161792755, "layer": 7, "max_source_mass": 0.5271762013435364, "mixer_index": 13, "normalized_entropy": 0.874313425279704, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5271762013435364, "median": 0.5114221572875977, "p05": 0.3548204302787781, "p95": 0.7728348970413208 }, { "label": "completed_group_1", "mean": 0.1581989824771881, "median": 0.1618250608444214, "p05": 0.07541970163583755, "p95": 0.21691158413887024 }, { "label": "current_group_partial", "mean": 0.3146248161792755, "median": 0.32365772128105164, "p05": 0.14578065276145935, "p95": 0.4483411908149719 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0704245567321777, "group": 2, "latest_source_mass": 0.38260507583618164, "layer": 8, "max_source_mass": 0.38260507583618164, "mixer_index": 14, "normalized_entropy": 0.9743424206822726, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3788670599460602, "median": 0.39538437128067017, "p05": 0.27489906549453735, "p95": 0.43731769919395447 }, { "label": "completed_group_1", "mean": 0.23852786421775818, "median": 0.2294245958328247, "p05": 0.20573674142360687, "p95": 0.29735344648361206 }, { "label": "current_group_partial", "mean": 0.38260507583618164, "median": 0.37669745087623596, "p05": 0.34335845708847046, "p95": 0.435106486082077 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9674104452133179, "group": 2, "latest_source_mass": 0.3281455934047699, "layer": 8, "max_source_mass": 0.48829177021980286, "mixer_index": 15, "normalized_entropy": 0.8805749354816949, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.48829177021980286, "median": 0.44956350326538086, "p05": 0.2823859453201294, "p95": 0.8209846615791321 }, { "label": "completed_group_1", "mean": 0.18356263637542725, "median": 0.18210050463676453, "p05": 0.05149933695793152, "p95": 0.2906108498573303 }, { "label": "current_group_partial", "mean": 0.3281455934047699, "median": 0.34843379259109497, "p05": 0.12376503646373749, "p95": 0.46937939524650574 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0832805633544922, "group": 3, "latest_source_mass": 0.31498202681541443, "layer": 9, "max_source_mass": 0.40388280153274536, "mixer_index": 16, "normalized_entropy": 0.9860444622076776, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.40388280153274536, "median": 0.403384804725647, "p05": 0.3489384353160858, "p95": 0.4601682126522064 }, { "label": "completed_group_1", "mean": 0.2811351716518402, "median": 0.28186845779418945, "p05": 0.25417307019233704, "p95": 0.30605652928352356 }, { "label": "completed_group_2", "mean": 0.31498202681541443, "median": 0.3153840899467468, "p05": 0.2613564431667328, "p95": 0.3658811151981354 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2917464971542358, "group": 3, "latest_source_mass": 0.33769795298576355, "layer": 9, "max_source_mass": 0.33769795298576355, "mixer_index": 17, "normalized_entropy": 0.9317981327650527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3349480926990509, "median": 0.3352771997451782, "p05": 0.18474003672599792, "p95": 0.48752525448799133 }, { "label": "completed_group_1", "mean": 0.1244347020983696, "median": 0.11896269023418427, "p05": 0.07986846566200256, "p95": 0.1843445897102356 }, { "label": "completed_group_2", "mean": 0.20291924476623535, "median": 0.20060715079307556, "p05": 0.1253921389579773, "p95": 0.2836313247680664 }, { "label": "current_group_partial", "mean": 0.33769795298576355, "median": 0.33908897638320923, "p05": 0.26979348063468933, "p95": 0.4016563892364502 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3533791303634644, "group": 3, "latest_source_mass": 0.28473395109176636, "layer": 10, "max_source_mass": 0.32115638256073, "mixer_index": 18, "normalized_entropy": 0.976256679908994, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.32115638256073, "median": 0.3355404734611511, "p05": 0.22348493337631226, "p95": 0.3863494396209717 }, { "label": "completed_group_1", "mean": 0.20602071285247803, "median": 0.2052205502986908, "p05": 0.1802767962217331, "p95": 0.23218519985675812 }, { "label": "completed_group_2", "mean": 0.18808895349502563, "median": 0.18706050515174866, "p05": 0.16633892059326172, "p95": 0.2130376398563385 }, { "label": "current_group_partial", "mean": 0.28473395109176636, "median": 0.275795578956604, "p05": 0.22748424112796783, "p95": 0.3660258948802948 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.308367371559143, "group": 3, "latest_source_mass": 0.29719752073287964, "layer": 10, "max_source_mass": 0.3113970160484314, "mixer_index": 19, "normalized_entropy": 0.9437875593046517, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3113970160484314, "median": 0.33913683891296387, "p05": 0.0915825292468071, "p95": 0.47554588317871094 }, { "label": "completed_group_1", "mean": 0.16179001331329346, "median": 0.16501447558403015, "p05": 0.10682816803455353, "p95": 0.20702867209911346 }, { "label": "completed_group_2", "mean": 0.2296154499053955, "median": 0.2261994481086731, "p05": 0.14510810375213623, "p95": 0.32595163583755493 }, { "label": "current_group_partial", "mean": 0.29719752073287964, "median": 0.2761377692222595, "p05": 0.194716677069664, "p95": 0.4515363574028015 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3437637090682983, "group": 3, "latest_source_mass": 0.3098427951335907, "layer": 11, "max_source_mass": 0.3217787742614746, "mixer_index": 20, "normalized_entropy": 0.969320619599697, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3217787742614746, "median": 0.31754863262176514, "p05": 0.25967735052108765, "p95": 0.40468457341194153 }, { "label": "completed_group_1", "mean": 0.1920534074306488, "median": 0.1915990114212036, "p05": 0.16391108930110931, "p95": 0.22057634592056274 }, { "label": "completed_group_2", "mean": 0.1763250231742859, "median": 0.1779440939426422, "p05": 0.14714840054512024, "p95": 0.1996835470199585 }, { "label": "current_group_partial", "mean": 0.3098427951335907, "median": 0.31089526414871216, "p05": 0.2459525167942047, "p95": 0.36900240182876587 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2280659675598145, "group": 3, "latest_source_mass": 0.29383569955825806, "layer": 11, "max_source_mass": 0.42534634470939636, "mixer_index": 21, "normalized_entropy": 0.8858623406415255, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.42534634470939636, "median": 0.4574331045150757, "p05": 0.17354995012283325, "p95": 0.6023476123809814 }, { "label": "completed_group_1", "mean": 0.14848899841308594, "median": 0.14150214195251465, "p05": 0.07509882748126984, "p95": 0.23531809449195862 }, { "label": "completed_group_2", "mean": 0.13232898712158203, "median": 0.13064970076084137, "p05": 0.07000484317541122, "p95": 0.19124652445316315 }, { "label": "current_group_partial", "mean": 0.29383569955825806, "median": 0.27858495712280273, "p05": 0.187642902135849, "p95": 0.4577033817768097 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3462648391723633, "group": 3, "latest_source_mass": 0.3101534843444824, "layer": 12, "max_source_mass": 0.3221249580383301, "mixer_index": 22, "normalized_entropy": 0.9711248035985732, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3221249580383301, "median": 0.3120178282260895, "p05": 0.2763444483280182, "p95": 0.3882659375667572 }, { "label": "completed_group_1", "mean": 0.18557845056056976, "median": 0.1851869523525238, "p05": 0.16533124446868896, "p95": 0.20681798458099365 }, { "label": "completed_group_2", "mean": 0.18214309215545654, "median": 0.18278372287750244, "p05": 0.15564095973968506, "p95": 0.2072875201702118 }, { "label": "current_group_partial", "mean": 0.3101534843444824, "median": 0.31112557649612427, "p05": 0.2608441412448883, "p95": 0.36039745807647705 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.147221326828003, "group": 3, "latest_source_mass": 0.2585829496383667, "layer": 12, "max_source_mass": 0.5053804516792297, "mixer_index": 23, "normalized_entropy": 0.8275452595084083, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5053804516792297, "median": 0.48097139596939087, "p05": 0.30125147104263306, "p95": 0.7477225065231323 }, { "label": "completed_group_1", "mean": 0.12089012563228607, "median": 0.11902710795402527, "p05": 0.05662901699542999, "p95": 0.1844676434993744 }, { "label": "completed_group_2", "mean": 0.11514648050069809, "median": 0.11994394659996033, "p05": 0.05982710421085358, "p95": 0.16130289435386658 }, { "label": "current_group_partial", "mean": 0.2585829496383667, "median": 0.25007814168930054, "p05": 0.12433226406574249, "p95": 0.42273372411727905 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3729398250579834, "group": 4, "latest_source_mass": 0.22271886467933655, "layer": 13, "max_source_mass": 0.30785465240478516, "mixer_index": 24, "normalized_entropy": 0.9903667385250569, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.30785465240478516, "median": 0.3129041790962219, "p05": 0.25217515230178833, "p95": 0.3537779450416565 }, { "label": "completed_group_1", "mean": 0.23079943656921387, "median": 0.22906282544136047, "p05": 0.19741898775100708, "p95": 0.26931890845298767 }, { "label": "completed_group_2", "mean": 0.23862707614898682, "median": 0.23815912008285522, "p05": 0.21215641498565674, "p95": 0.2663886845111847 }, { "label": "completed_group_3", "mean": 0.22271886467933655, "median": 0.22075486183166504, "p05": 0.186451256275177, "p95": 0.2638319730758667 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5232064723968506, "group": 4, "latest_source_mass": 0.24461600184440613, "layer": 13, "max_source_mass": 0.3136816620826721, "mixer_index": 25, "normalized_entropy": 0.9464213938474744, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3136816620826721, "median": 0.326117604970932, "p05": 0.20238417387008667, "p95": 0.4362232983112335 }, { "label": "completed_group_1", "mean": 0.11285123229026794, "median": 0.11373943835496902, "p05": 0.08906605839729309, "p95": 0.13395145535469055 }, { "label": "completed_group_2", "mean": 0.13196846842765808, "median": 0.1334267258644104, "p05": 0.10183953493833542, "p95": 0.16085752844810486 }, { "label": "completed_group_3", "mean": 0.19688265025615692, "median": 0.1940557211637497, "p05": 0.13830327987670898, "p95": 0.2671412229537964 }, { "label": "current_group_partial", "mean": 0.24461600184440613, "median": 0.2419893592596054, "p05": 0.18994617462158203, "p95": 0.3077915906906128 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4876129627227783, "group": 4, "latest_source_mass": 0.396677166223526, "layer": 14, "max_source_mass": 0.396677166223526, "mixer_index": 26, "normalized_entropy": 0.9243059028433878, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2035508155822754, "median": 0.20008960366249084, "p05": 0.16096048057079315, "p95": 0.255715548992157 }, { "label": "completed_group_1", "mean": 0.15372203290462494, "median": 0.15584711730480194, "p05": 0.12048184871673584, "p95": 0.18010932207107544 }, { "label": "completed_group_2", "mean": 0.13167491555213928, "median": 0.13113251328468323, "p05": 0.11609883606433868, "p95": 0.14900363981723785 }, { "label": "completed_group_3", "mean": 0.11437506973743439, "median": 0.11201842874288559, "p05": 0.09515875577926636, "p95": 0.1399364471435547 }, { "label": "current_group_partial", "mean": 0.396677166223526, "median": 0.39533013105392456, "p05": 0.3409242630004883, "p95": 0.4550648033618927 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4587602615356445, "group": 4, "latest_source_mass": 0.33461427688598633, "layer": 14, "max_source_mass": 0.33461427688598633, "mixer_index": 27, "normalized_entropy": 0.906378711639412, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28519001603126526, "median": 0.28675830364227295, "p05": 0.1500462144613266, "p95": 0.45491817593574524 }, { "label": "completed_group_1", "mean": 0.10532153397798538, "median": 0.09640030562877655, "p05": 0.06345808506011963, "p95": 0.1692449301481247 }, { "label": "completed_group_2", "mean": 0.09675674885511398, "median": 0.09718051552772522, "p05": 0.058796193450689316, "p95": 0.13363884389400482 }, { "label": "completed_group_3", "mean": 0.17811740934848785, "median": 0.1759425848722458, "p05": 0.1284305453300476, "p95": 0.23726706206798553 }, { "label": "current_group_partial", "mean": 0.33461427688598633, "median": 0.33546411991119385, "p05": 0.23162710666656494, "p95": 0.4484652876853943 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4565696716308594, "group": 4, "latest_source_mass": 0.4221859574317932, "layer": 15, "max_source_mass": 0.4221859574317932, "mixer_index": 28, "normalized_entropy": 0.9050176216042753, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20232918858528137, "median": 0.1998670995235443, "p05": 0.15170496702194214, "p95": 0.26285088062286377 }, { "label": "completed_group_1", "mean": 0.14928826689720154, "median": 0.14749380946159363, "p05": 0.12863489985466003, "p95": 0.17874351143836975 }, { "label": "completed_group_2", "mean": 0.12029382586479187, "median": 0.11950792372226715, "p05": 0.10211217403411865, "p95": 0.14088989794254303 }, { "label": "completed_group_3", "mean": 0.1059027910232544, "median": 0.10411787033081055, "p05": 0.08704904466867447, "p95": 0.13138693571090698 }, { "label": "current_group_partial", "mean": 0.4221859574317932, "median": 0.4195709228515625, "p05": 0.35523390769958496, "p95": 0.4969034492969513 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4788098335266113, "group": 4, "latest_source_mass": 0.34753501415252686, "layer": 15, "max_source_mass": 0.34753501415252686, "mixer_index": 29, "normalized_entropy": 0.9188362111403675, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2697942852973938, "median": 0.25651901960372925, "p05": 0.146310955286026, "p95": 0.42084938287734985 }, { "label": "completed_group_1", "mean": 0.11948063224554062, "median": 0.11549928039312363, "p05": 0.0849025622010231, "p95": 0.16021209955215454 }, { "label": "completed_group_2", "mean": 0.11090938746929169, "median": 0.11184529960155487, "p05": 0.08139486610889435, "p95": 0.13886672258377075 }, { "label": "completed_group_3", "mean": 0.15228068828582764, "median": 0.15164361894130707, "p05": 0.11261995136737823, "p95": 0.19522227346897125 }, { "label": "current_group_partial", "mean": 0.34753501415252686, "median": 0.3453425168991089, "p05": 0.25332537293434143, "p95": 0.44237542152404785 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5018762350082397, "group": 4, "latest_source_mass": 0.31088152527809143, "layer": 16, "max_source_mass": 0.31088152527809143, "mixer_index": 30, "normalized_entropy": 0.9331681721954809, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.208971306681633, "median": 0.20881521701812744, "p05": 0.16508029401302338, "p95": 0.2516583502292633 }, { "label": "completed_group_1", "mean": 0.27331358194351196, "median": 0.26948875188827515, "p05": 0.23055149614810944, "p95": 0.33120542764663696 }, { "label": "completed_group_2", "mean": 0.12766750156879425, "median": 0.1261082887649536, "p05": 0.10924770683050156, "p95": 0.1494429111480713 }, { "label": "completed_group_3", "mean": 0.07916608452796936, "median": 0.07811431586742401, "p05": 0.0561223030090332, "p95": 0.10760308802127838 }, { "label": "current_group_partial", "mean": 0.31088152527809143, "median": 0.3123162090778351, "p05": 0.2707694172859192, "p95": 0.3467172384262085 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5627843141555786, "group": 4, "latest_source_mass": 0.22652018070220947, "layer": 16, "max_source_mass": 0.27452588081359863, "mixer_index": 31, "normalized_entropy": 0.9710124895666443, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27452588081359863, "median": 0.2678954601287842, "p05": 0.19414913654327393, "p95": 0.3916982412338257 }, { "label": "completed_group_1", "mean": 0.21448183059692383, "median": 0.21617943048477173, "p05": 0.17793960869312286, "p95": 0.24279263615608215 }, { "label": "completed_group_2", "mean": 0.143874391913414, "median": 0.1446641981601715, "p05": 0.11340779811143875, "p95": 0.17275942862033844 }, { "label": "completed_group_3", "mean": 0.14059773087501526, "median": 0.13659776747226715, "p05": 0.11039599776268005, "p95": 0.18301524221897125 }, { "label": "current_group_partial", "mean": 0.22652018070220947, "median": 0.22071748971939087, "p05": 0.16668984293937683, "p95": 0.293569415807724 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.580754280090332, "group": 5, "latest_source_mass": 0.24107450246810913, "layer": 17, "max_source_mass": 0.2518634498119354, "mixer_index": 32, "normalized_entropy": 0.9821778571747528, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2518634498119354, "median": 0.24593469500541687, "p05": 0.21242231130599976, "p95": 0.3016672730445862 }, { "label": "completed_group_1", "mean": 0.18758200109004974, "median": 0.1801876276731491, "p05": 0.15723729133605957, "p95": 0.24892441928386688 }, { "label": "completed_group_2", "mean": 0.13903990387916565, "median": 0.13839681446552277, "p05": 0.12513971328735352, "p95": 0.15491701662540436 }, { "label": "completed_group_3", "mean": 0.18044015765190125, "median": 0.1819951832294464, "p05": 0.14810611307621002, "p95": 0.20801132917404175 }, { "label": "completed_group_4", "mean": 0.24107450246810913, "median": 0.24384509027004242, "p05": 0.1871698796749115, "p95": 0.2886185050010681 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7564777135849, "group": 5, "latest_source_mass": 0.2205006182193756, "layer": 17, "max_source_mass": 0.2205006182193756, "mixer_index": 33, "normalized_entropy": 0.9803088772521708, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15440437197685242, "median": 0.15525281429290771, "p05": 0.11778812110424042, "p95": 0.18865147233009338 }, { "label": "completed_group_1", "mean": 0.19411899149417877, "median": 0.2002902328968048, "p05": 0.14482440054416656, "p95": 0.2256385236978531 }, { "label": "completed_group_2", "mean": 0.17806550860404968, "median": 0.1781877875328064, "p05": 0.15101563930511475, "p95": 0.20418882369995117 }, { "label": "completed_group_3", "mean": 0.14696575701236725, "median": 0.14620032906532288, "p05": 0.10836094617843628, "p95": 0.1874053031206131 }, { "label": "completed_group_4", "mean": 0.10594475269317627, "median": 0.10578415542840958, "p05": 0.08951808512210846, "p95": 0.1216079369187355 }, { "label": "current_group_partial", "mean": 0.2205006182193756, "median": 0.21638520061969757, "p05": 0.16007332503795624, "p95": 0.2944811284542084 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.5899518728256226, "group": 5, "latest_source_mass": 0.4157395362854004, "layer": 18, "max_source_mass": 0.4157395362854004, "mixer_index": 34, "normalized_entropy": 0.8873690359290373, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1636897772550583, "median": 0.16218292713165283, "p05": 0.1178273856639862, "p95": 0.21766462922096252 }, { "label": "completed_group_1", "mean": 0.11789833009243011, "median": 0.11790023744106293, "p05": 0.1032494530081749, "p95": 0.13267967104911804 }, { "label": "completed_group_2", "mean": 0.07030131667852402, "median": 0.0698043555021286, "p05": 0.06258689612150192, "p95": 0.07938572019338608 }, { "label": "completed_group_3", "mean": 0.09238425642251968, "median": 0.09013497084379196, "p05": 0.0768275037407875, "p95": 0.1164383590221405 }, { "label": "completed_group_4", "mean": 0.1399867981672287, "median": 0.1400289535522461, "p05": 0.12679682672023773, "p95": 0.15317322313785553 }, { "label": "current_group_partial", "mean": 0.4157395362854004, "median": 0.4171099066734314, "p05": 0.367803692817688, "p95": 0.4605288803577423 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.2894535064697266, "group": 5, "latest_source_mass": 0.5977407693862915, "layer": 18, "max_source_mass": 0.5977407693862915, "mixer_index": 35, "normalized_entropy": 0.7196577044045218, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1212274506688118, "median": 0.10892187058925629, "p05": 0.07249757647514343, "p95": 0.2145335078239441 }, { "label": "completed_group_1", "mean": 0.06944876164197922, "median": 0.0678764283657074, "p05": 0.05350388586521149, "p95": 0.08999023586511612 }, { "label": "completed_group_2", "mean": 0.05726535990834236, "median": 0.0559837631881237, "p05": 0.0409373864531517, "p95": 0.07825973629951477 }, { "label": "completed_group_3", "mean": 0.09096579253673553, "median": 0.08796361088752747, "p05": 0.06407731771469116, "p95": 0.12661314010620117 }, { "label": "completed_group_4", "mean": 0.06335186958312988, "median": 0.06283852458000183, "p05": 0.05032140761613846, "p95": 0.0781375840306282 }, { "label": "current_group_partial", "mean": 0.5977407693862915, "median": 0.6085352301597595, "p05": 0.4878399074077606, "p95": 0.6737192869186401 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6581614017486572, "group": 5, "latest_source_mass": 0.35773903131484985, "layer": 19, "max_source_mass": 0.35773903131484985, "mixer_index": 36, "normalized_entropy": 0.9254374988530375, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18132120370864868, "median": 0.1806161105632782, "p05": 0.1410265862941742, "p95": 0.22331592440605164 }, { "label": "completed_group_1", "mean": 0.12711825966835022, "median": 0.12722507119178772, "p05": 0.11291128396987915, "p95": 0.14178316295146942 }, { "label": "completed_group_2", "mean": 0.08560527861118317, "median": 0.08513904362916946, "p05": 0.07705061137676239, "p95": 0.09564155340194702 }, { "label": "completed_group_3", "mean": 0.09748885035514832, "median": 0.0953599065542221, "p05": 0.08346174657344818, "p95": 0.1194010004401207 }, { "label": "completed_group_4", "mean": 0.15072739124298096, "median": 0.150638610124588, "p05": 0.13762864470481873, "p95": 0.1634974181652069 }, { "label": "current_group_partial", "mean": 0.35773903131484985, "median": 0.35794761776924133, "p05": 0.32225728034973145, "p95": 0.39409786462783813 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3699336051940918, "group": 5, "latest_source_mass": 0.5385624766349792, "layer": 19, "max_source_mass": 0.5385624766349792, "mixer_index": 37, "normalized_entropy": 0.7645745027284836, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13604649901390076, "median": 0.11998516321182251, "p05": 0.06740576773881912, "p95": 0.26897257566452026 }, { "label": "completed_group_1", "mean": 0.10151474922895432, "median": 0.0991508960723877, "p05": 0.07438431680202484, "p95": 0.13600951433181763 }, { "label": "completed_group_2", "mean": 0.07715044170618057, "median": 0.0767412930727005, "p05": 0.054624155163764954, "p95": 0.09943743050098419 }, { "label": "completed_group_3", "mean": 0.11300846189260483, "median": 0.11003986746072769, "p05": 0.08064281940460205, "p95": 0.1516837179660797 }, { "label": "completed_group_4", "mean": 0.033717282116413116, "median": 0.03286907076835632, "p05": 0.024589763954281807, "p95": 0.04592139273881912 }, { "label": "current_group_partial", "mean": 0.5385624766349792, "median": 0.5415897369384766, "p05": 0.4144754409790039, "p95": 0.6506978273391724 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6926933526992798, "group": 5, "latest_source_mass": 0.3198680281639099, "layer": 20, "max_source_mass": 0.3198680281639099, "mixer_index": 38, "normalized_entropy": 0.9447101476341264, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19425281882286072, "median": 0.19403350353240967, "p05": 0.139950692653656, "p95": 0.2494640350341797 }, { "label": "completed_group_1", "mean": 0.14001017808914185, "median": 0.14058321714401245, "p05": 0.12015820294618607, "p95": 0.158815398812294 }, { "label": "completed_group_2", "mean": 0.09434760361909866, "median": 0.09424005448818207, "p05": 0.08333250135183334, "p95": 0.10548526048660278 }, { "label": "completed_group_3", "mean": 0.10331346094608307, "median": 0.10153411328792572, "p05": 0.08690286427736282, "p95": 0.12658309936523438 }, { "label": "completed_group_4", "mean": 0.14820793271064758, "median": 0.14894580841064453, "p05": 0.13278263807296753, "p95": 0.16199614107608795 }, { "label": "current_group_partial", "mean": 0.3198680281639099, "median": 0.3201481103897095, "p05": 0.2800636887550354, "p95": 0.3604968786239624 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4148362874984741, "group": 5, "latest_source_mass": 0.4996800422668457, "layer": 20, "max_source_mass": 0.4996800422668457, "mixer_index": 39, "normalized_entropy": 0.789635166883214, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16652248799800873, "median": 0.14292314648628235, "p05": 0.08186839520931244, "p95": 0.350705623626709 }, { "label": "completed_group_1", "mean": 0.10329679399728775, "median": 0.10057851672172546, "p05": 0.07478320598602295, "p95": 0.13858266174793243 }, { "label": "completed_group_2", "mean": 0.08063912391662598, "median": 0.0807541012763977, "p05": 0.056308042258024216, "p95": 0.10469165444374084 }, { "label": "completed_group_3", "mean": 0.1165856122970581, "median": 0.11373114585876465, "p05": 0.08339665830135345, "p95": 0.15619920194149017 }, { "label": "completed_group_4", "mean": 0.03327595070004463, "median": 0.032142095267772675, "p05": 0.022759035229682922, "p95": 0.04896518588066101 }, { "label": "current_group_partial", "mean": 0.4996800422668457, "median": 0.5001581907272339, "p05": 0.3463003933429718, "p95": 0.6266770362854004 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.740808129310608, "group": 6, "latest_source_mass": 0.24231041967868805, "layer": 21, "max_source_mass": 0.24231041967868805, "mixer_index": 40, "normalized_entropy": 0.9715635157550481, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21459883451461792, "median": 0.21599377691745758, "p05": 0.1570315659046173, "p95": 0.27208006381988525 }, { "label": "completed_group_1", "mean": 0.15640269219875336, "median": 0.1569681316614151, "p05": 0.13785089552402496, "p95": 0.1749057024717331 }, { "label": "completed_group_2", "mean": 0.10570268332958221, "median": 0.10564640164375305, "p05": 0.09223169088363647, "p95": 0.11907710134983063 }, { "label": "completed_group_3", "mean": 0.11097013205289841, "median": 0.10862545669078827, "p05": 0.09454305469989777, "p95": 0.13635265827178955 }, { "label": "completed_group_4", "mean": 0.17001524567604065, "median": 0.17010903358459473, "p05": 0.15088173747062683, "p95": 0.18767674267292023 }, { "label": "completed_group_5", "mean": 0.24231041967868805, "median": 0.24274763464927673, "p05": 0.2087050974369049, "p95": 0.27555277943611145 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3179612159729004, "group": 6, "latest_source_mass": 0.603651762008667, "layer": 21, "max_source_mass": 0.603651762008667, "mixer_index": 41, "normalized_entropy": 0.6772980841960945, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08068332821130753, "median": 0.06917878985404968, "p05": 0.039396293461322784, "p95": 0.17595769464969635 }, { "label": "completed_group_1", "mean": 0.049232251942157745, "median": 0.04861311614513397, "p05": 0.03742668777704239, "p95": 0.06363335251808167 }, { "label": "completed_group_2", "mean": 0.045073024928569794, "median": 0.04482679069042206, "p05": 0.03333880379796028, "p95": 0.057495348155498505 }, { "label": "completed_group_3", "mean": 0.06111945956945419, "median": 0.059528060257434845, "p05": 0.042710162699222565, "p95": 0.08362975716590881 }, { "label": "completed_group_4", "mean": 0.028317855671048164, "median": 0.027924545109272003, "p05": 0.022549759596586227, "p95": 0.03542562201619148 }, { "label": "completed_group_5", "mean": 0.13192236423492432, "median": 0.12939827144145966, "p05": 0.10056710243225098, "p95": 0.17195004224777222 }, { "label": "current_group_partial", "mean": 0.603651762008667, "median": 0.6094026565551758, "p05": 0.5065152645111084, "p95": 0.6787958145141602 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.5319744348526, "group": 6, "latest_source_mass": 0.5132461190223694, "layer": 22, "max_source_mass": 0.5132461190223694, "mixer_index": 42, "normalized_entropy": 0.7872791226235868, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12406684458255768, "median": 0.1289612352848053, "p05": 0.06980825215578079, "p95": 0.16972985863685608 }, { "label": "completed_group_1", "mean": 0.07462044060230255, "median": 0.07392798364162445, "p05": 0.06016036123037338, "p95": 0.0917147696018219 }, { "label": "completed_group_2", "mean": 0.05378449708223343, "median": 0.053309448063373566, "p05": 0.04370884597301483, "p95": 0.06546851992607117 }, { "label": "completed_group_3", "mean": 0.05575074627995491, "median": 0.053552836179733276, "p05": 0.0437602773308754, "p95": 0.07444392144680023 }, { "label": "completed_group_4", "mean": 0.07357081025838852, "median": 0.07329098880290985, "p05": 0.06283499300479889, "p95": 0.08542977273464203 }, { "label": "completed_group_5", "mean": 0.10496053099632263, "median": 0.1038457602262497, "p05": 0.08522139489650726, "p95": 0.1291121542453766 }, { "label": "current_group_partial", "mean": 0.5132461190223694, "median": 0.5090781450271606, "p05": 0.4398295283317566, "p95": 0.59932541847229 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.2461028099060059, "group": 6, "latest_source_mass": 0.6375080347061157, "layer": 22, "max_source_mass": 0.6375080347061157, "mixer_index": 43, "normalized_entropy": 0.640370168432985, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08556011319160461, "median": 0.07947102189064026, "p05": 0.043556369841098785, "p95": 0.16172736883163452 }, { "label": "completed_group_1", "mean": 0.0446893647313118, "median": 0.04398339241743088, "p05": 0.03128381073474884, "p95": 0.05990619957447052 }, { "label": "completed_group_2", "mean": 0.04084867238998413, "median": 0.04042873531579971, "p05": 0.02942952886223793, "p95": 0.05392487719655037 }, { "label": "completed_group_3", "mean": 0.05386577919125557, "median": 0.052273523062467575, "p05": 0.033789411187171936, "p95": 0.076903335750103 }, { "label": "completed_group_4", "mean": 0.025119207799434662, "median": 0.024625632911920547, "p05": 0.019138969480991364, "p95": 0.03265690058469772 }, { "label": "completed_group_5", "mean": 0.11240880191326141, "median": 0.11003861576318741, "p05": 0.07852104306221008, "p95": 0.155371755361557 }, { "label": "current_group_partial", "mean": 0.6375080347061157, "median": 0.6392797231674194, "p05": 0.5317062735557556, "p95": 0.7386638522148132 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.6323832273483276, "group": 6, "latest_source_mass": 0.4532663822174072, "layer": 23, "max_source_mass": 0.4532663822174072, "mixer_index": 44, "normalized_entropy": 0.8388790346464895, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1418324112892151, "median": 0.14852997660636902, "p05": 0.07787702977657318, "p95": 0.1923113763332367 }, { "label": "completed_group_1", "mean": 0.08991365134716034, "median": 0.08918912708759308, "p05": 0.0752246081829071, "p95": 0.10654127597808838 }, { "label": "completed_group_2", "mean": 0.06188817322254181, "median": 0.06161077320575714, "p05": 0.04994015768170357, "p95": 0.07481888681650162 }, { "label": "completed_group_3", "mean": 0.05739111453294754, "median": 0.0548475980758667, "p05": 0.04428064078092575, "p95": 0.07816634327173233 }, { "label": "completed_group_4", "mean": 0.09848874807357788, "median": 0.09768162667751312, "p05": 0.08320866525173187, "p95": 0.11606857925653458 }, { "label": "completed_group_5", "mean": 0.0972195416688919, "median": 0.09613433480262756, "p05": 0.07767169177532196, "p95": 0.12004032731056213 }, { "label": "current_group_partial", "mean": 0.4532663822174072, "median": 0.45108917355537415, "p05": 0.37653055787086487, "p95": 0.5389904975891113 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.301480770111084, "group": 6, "latest_source_mass": 0.6134276390075684, "layer": 23, "max_source_mass": 0.6134276390075684, "mixer_index": 45, "normalized_entropy": 0.6688288103861927, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09513165056705475, "median": 0.08380547165870667, "p05": 0.055099427700042725, "p95": 0.1882638931274414 }, { "label": "completed_group_1", "mean": 0.050291214138269424, "median": 0.04947858303785324, "p05": 0.03757611662149429, "p95": 0.06516233086585999 }, { "label": "completed_group_2", "mean": 0.045042067766189575, "median": 0.0443364642560482, "p05": 0.031873252242803574, "p95": 0.06115761026740074 }, { "label": "completed_group_3", "mean": 0.05902485549449921, "median": 0.05624900758266449, "p05": 0.038259170949459076, "p95": 0.08772744238376617 }, { "label": "completed_group_4", "mean": 0.02615261636674404, "median": 0.02565642073750496, "p05": 0.01966264471411705, "p95": 0.03415588289499283 }, { "label": "completed_group_5", "mean": 0.1109299436211586, "median": 0.10745325684547424, "p05": 0.07642066478729248, "p95": 0.15984633564949036 }, { "label": "current_group_partial", "mean": 0.6134276390075684, "median": 0.6200529336929321, "p05": 0.49638131260871887, "p95": 0.7104787826538086 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.629331111907959, "group": 6, "latest_source_mass": 0.45539987087249756, "layer": 24, "max_source_mass": 0.45539987087249756, "mixer_index": 46, "normalized_entropy": 0.837310557580963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14790503680706024, "median": 0.15420395135879517, "p05": 0.08257082104682922, "p95": 0.20396333932876587 }, { "label": "completed_group_1", "mean": 0.08658000826835632, "median": 0.08578156679868698, "p05": 0.07299856841564178, "p95": 0.10278668999671936 }, { "label": "completed_group_2", "mean": 0.06608415395021439, "median": 0.06551258265972137, "p05": 0.054121918976306915, "p95": 0.07912646979093552 }, { "label": "completed_group_3", "mean": 0.06427294760942459, "median": 0.06133077293634415, "p05": 0.04914242774248123, "p95": 0.09066928923130035 }, { "label": "completed_group_4", "mean": 0.0819322019815445, "median": 0.08114765584468842, "p05": 0.06886082887649536, "p95": 0.09821749478578568 }, { "label": "completed_group_5", "mean": 0.0978257954120636, "median": 0.09643188118934631, "p05": 0.07740232348442078, "p95": 0.12209523469209671 }, { "label": "current_group_partial", "mean": 0.45539987087249756, "median": 0.45274412631988525, "p05": 0.3780696392059326, "p95": 0.5462272763252258 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.338958501815796, "group": 6, "latest_source_mass": 0.5946891903877258, "layer": 24, "max_source_mass": 0.5946891903877258, "mixer_index": 47, "normalized_entropy": 0.6880885545850224, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10803166776895523, "median": 0.09507323801517487, "p05": 0.04510519653558731, "p95": 0.2335781455039978 }, { "label": "completed_group_1", "mean": 0.0559723861515522, "median": 0.05525296553969383, "p05": 0.042058296501636505, "p95": 0.07207933813333511 }, { "label": "completed_group_2", "mean": 0.047874726355075836, "median": 0.04723185673356056, "p05": 0.03294869884848595, "p95": 0.06497423350811005 }, { "label": "completed_group_3", "mean": 0.060293328016996384, "median": 0.05662515014410019, "p05": 0.040346790105104446, "p95": 0.08978720009326935 }, { "label": "completed_group_4", "mean": 0.029215168207883835, "median": 0.028496116399765015, "p05": 0.021991699934005737, "p95": 0.039112430065870285 }, { "label": "completed_group_5", "mean": 0.10392352938652039, "median": 0.10063231736421585, "p05": 0.06900570541620255, "p95": 0.1502695530653 }, { "label": "current_group_partial", "mean": 0.5946891903877258, "median": 0.600748598575592, "p05": 0.46449366211891174, "p95": 0.7016042470932007 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7611865997314453, "group": 7, "latest_source_mass": 0.3679775297641754, "layer": 25, "max_source_mass": 0.3679775297641754, "mixer_index": 48, "normalized_entropy": 0.9050708742058073, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1573912650346756, "median": 0.15898014605045319, "p05": 0.09474922716617584, "p95": 0.2298184633255005 }, { "label": "completed_group_1", "mean": 0.10317619889974594, "median": 0.10270150750875473, "p05": 0.08812867850065231, "p95": 0.11994096636772156 }, { "label": "completed_group_2", "mean": 0.08533225208520889, "median": 0.08495655655860901, "p05": 0.07036984711885452, "p95": 0.1013721376657486 }, { "label": "completed_group_3", "mean": 0.08218386024236679, "median": 0.07942138612270355, "p05": 0.06530377268791199, "p95": 0.1088358536362648 }, { "label": "completed_group_4", "mean": 0.09052872657775879, "median": 0.08931052684783936, "p05": 0.07639768719673157, "p95": 0.10804671049118042 }, { "label": "completed_group_5", "mean": 0.11341015249490738, "median": 0.1123356968164444, "p05": 0.09177497029304504, "p95": 0.13803066313266754 }, { "label": "completed_group_6", "mean": 0.3679775297641754, "median": 0.3671194612979889, "p05": 0.2961871325969696, "p95": 0.4462817311286926 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.476489782333374, "group": 7, "latest_source_mass": 0.3396714925765991, "layer": 25, "max_source_mass": 0.3836256265640259, "mixer_index": 49, "normalized_entropy": 0.7100414956318613, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09070298075675964, "median": 0.0818760022521019, "p05": 0.04576021432876587, "p95": 0.1613437533378601 }, { "label": "completed_group_1", "mean": 0.03713828697800636, "median": 0.035560622811317444, "p05": 0.02337636426091194, "p95": 0.054472144693136215 }, { "label": "completed_group_2", "mean": 0.030013620853424072, "median": 0.028701743111014366, "p05": 0.01836019568145275, "p95": 0.04582716524600983 }, { "label": "completed_group_3", "mean": 0.03533655032515526, "median": 0.031870774924755096, "p05": 0.018502909690141678, "p95": 0.061826568096876144 }, { "label": "completed_group_4", "mean": 0.020730752497911453, "median": 0.020491186529397964, "p05": 0.014567261561751366, "p95": 0.027856238186359406 }, { "label": "completed_group_5", "mean": 0.06278066337108612, "median": 0.059403590857982635, "p05": 0.03608490154147148, "p95": 0.10122650861740112 }, { "label": "completed_group_6", "mean": 0.3836256265640259, "median": 0.3773127794265747, "p05": 0.1874689906835556, "p95": 0.5895293951034546 }, { "label": "current_group_partial", "mean": 0.3396714925765991, "median": 0.34821513295173645, "p05": 0.09212851524353027, "p95": 0.587702751159668 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7213938236236572, "group": 7, "latest_source_mass": 0.37924307584762573, "layer": 26, "max_source_mass": 0.37924307584762573, "mixer_index": 50, "normalized_entropy": 0.8278154442529139, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10584281384944916, "median": 0.10982948541641235, "p05": 0.03868259862065315, "p95": 0.16580678522586823 }, { "label": "completed_group_1", "mean": 0.06887699663639069, "median": 0.06925956159830093, "p05": 0.04082746431231499, "p95": 0.09546782076358795 }, { "label": "completed_group_2", "mean": 0.05207539349794388, "median": 0.051871635019779205, "p05": 0.035383015871047974, "p95": 0.07048650085926056 }, { "label": "completed_group_3", "mean": 0.050678279250860214, "median": 0.04831986129283905, "p05": 0.03378130868077278, "p95": 0.07452918589115143 }, { "label": "completed_group_4", "mean": 0.04768219590187073, "median": 0.04721469432115555, "p05": 0.03680850565433502, "p95": 0.060343313962221146 }, { "label": "completed_group_5", "mean": 0.06809806823730469, "median": 0.06714263558387756, "p05": 0.04759174585342407, "p95": 0.09223967790603638 }, { "label": "completed_group_6", "mean": 0.2275032103061676, "median": 0.22425106167793274, "p05": 0.14320722222328186, "p95": 0.3203037679195404 }, { "label": "current_group_partial", "mean": 0.37924307584762573, "median": 0.3716055750846863, "p05": 0.20950253307819366, "p95": 0.5737906098365784 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5405172109603882, "group": 7, "latest_source_mass": 0.4017985463142395, "layer": 26, "max_source_mass": 0.4017985463142395, "mixer_index": 51, "normalized_entropy": 0.7408321802188831, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.074287548661232, "median": 0.06164507195353508, "p05": 0.028203874826431274, "p95": 0.19368377327919006 }, { "label": "completed_group_1", "mean": 0.04491765797138214, "median": 0.04365704953670502, "p05": 0.02776508405804634, "p95": 0.06526201963424683 }, { "label": "completed_group_2", "mean": 0.035157643258571625, "median": 0.03418130800127983, "p05": 0.022541508078575134, "p95": 0.050758615136146545 }, { "label": "completed_group_3", "mean": 0.046252522617578506, "median": 0.04353209584951401, "p05": 0.026215802878141403, "p95": 0.07329559326171875 }, { "label": "completed_group_4", "mean": 0.023427214473485947, "median": 0.023055266588926315, "p05": 0.016644639894366264, "p95": 0.03150089830160141 }, { "label": "completed_group_5", "mean": 0.07749318331480026, "median": 0.07363636791706085, "p05": 0.04229677468538284, "p95": 0.12579958140850067 }, { "label": "completed_group_6", "mean": 0.29666566848754883, "median": 0.2920765280723572, "p05": 0.14379237592220306, "p95": 0.45965293049812317 }, { "label": "current_group_partial", "mean": 0.4017985463142395, "median": 0.4004032611846924, "p05": 0.18456703424453735, "p95": 0.6275562047958374 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8063910007476807, "group": 7, "latest_source_mass": 0.3437207341194153, "layer": 27, "max_source_mass": 0.3437207341194153, "mixer_index": 52, "normalized_entropy": 0.868690446228377, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11433833092451096, "median": 0.11714513599872589, "p05": 0.06115472689270973, "p95": 0.16258974373340607 }, { "label": "completed_group_1", "mean": 0.07374332845211029, "median": 0.0731307864189148, "p05": 0.05447638779878616, "p95": 0.09509895741939545 }, { "label": "completed_group_2", "mean": 0.05932536721229553, "median": 0.05929289758205414, "p05": 0.04452420026063919, "p95": 0.07498160749673843 }, { "label": "completed_group_3", "mean": 0.05389238893985748, "median": 0.051720209419727325, "p05": 0.03926049917936325, "p95": 0.07536239922046661 }, { "label": "completed_group_4", "mean": 0.06126017868518829, "median": 0.06108957529067993, "p05": 0.0506296306848526, "p95": 0.07238088548183441 }, { "label": "completed_group_5", "mean": 0.07349519431591034, "median": 0.07249070703983307, "p05": 0.05706139653921127, "p95": 0.09332700818777084 }, { "label": "completed_group_6", "mean": 0.22022445499897003, "median": 0.21959853172302246, "p05": 0.15368635952472687, "p95": 0.2904376685619354 }, { "label": "current_group_partial", "mean": 0.3437207341194153, "median": 0.33986932039260864, "p05": 0.22015926241874695, "p95": 0.4802429676055908 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.4859801530838013, "group": 7, "latest_source_mass": 0.4424910843372345, "layer": 27, "max_source_mass": 0.4424910843372345, "mixer_index": 53, "normalized_entropy": 0.7146053992378076, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08387120068073273, "median": 0.07644708454608917, "p05": 0.0386628657579422, "p95": 0.15876422822475433 }, { "label": "completed_group_1", "mean": 0.041406162083148956, "median": 0.0394824743270874, "p05": 0.025755178183317184, "p95": 0.06136181950569153 }, { "label": "completed_group_2", "mean": 0.032444946467876434, "median": 0.0307333804666996, "p05": 0.018246563151478767, "p95": 0.052875272929668427 }, { "label": "completed_group_3", "mean": 0.03686721250414848, "median": 0.03247762471437454, "p05": 0.017809681594371796, "p95": 0.06780916452407837 }, { "label": "completed_group_4", "mean": 0.025052767246961594, "median": 0.024561133235692978, "p05": 0.017057543620467186, "p95": 0.034822143614292145 }, { "label": "completed_group_5", "mean": 0.06012478470802307, "median": 0.056938059628009796, "p05": 0.03254516050219536, "p95": 0.09905251860618591 }, { "label": "completed_group_6", "mean": 0.27774184942245483, "median": 0.2747848629951477, "p05": 0.11654777824878693, "p95": 0.4562506675720215 }, { "label": "current_group_partial", "mean": 0.4424910843372345, "median": 0.4387660026550293, "p05": 0.21194897592067719, "p95": 0.6782112717628479 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7777256965637207, "group": 7, "latest_source_mass": 0.3409850299358368, "layer": 28, "max_source_mass": 0.3409850299358368, "mixer_index": 54, "normalized_entropy": 0.8549053488311195, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10586117208003998, "median": 0.10652337223291397, "p05": 0.05095779523253441, "p95": 0.1629783660173416 }, { "label": "completed_group_1", "mean": 0.07087038457393646, "median": 0.06906798481941223, "p05": 0.04985840618610382, "p95": 0.09773125499486923 }, { "label": "completed_group_2", "mean": 0.05850498005747795, "median": 0.05767729878425598, "p05": 0.04179685190320015, "p95": 0.07779045403003693 }, { "label": "completed_group_3", "mean": 0.06360965222120285, "median": 0.06068425625562668, "p05": 0.04235687851905823, "p95": 0.09371945261955261 }, { "label": "completed_group_4", "mean": 0.04672512412071228, "median": 0.04617916792631149, "p05": 0.0356140062212944, "p95": 0.05914818495512009 }, { "label": "completed_group_5", "mean": 0.07485505938529968, "median": 0.07274073362350464, "p05": 0.052147433161735535, "p95": 0.10333258658647537 }, { "label": "completed_group_6", "mean": 0.2385886013507843, "median": 0.2345724105834961, "p05": 0.15389031171798706, "p95": 0.33735349774360657 }, { "label": "current_group_partial", "mean": 0.3409850299358368, "median": 0.33667099475860596, "p05": 0.18655306100845337, "p95": 0.5089476704597473 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6318418979644775, "group": 7, "latest_source_mass": 0.3292362689971924, "layer": 28, "max_source_mass": 0.3292362689971924, "mixer_index": 55, "normalized_entropy": 0.7847500712360619, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09244748950004578, "median": 0.06782519817352295, "p05": 0.025312144309282303, "p95": 0.24087309837341309 }, { "label": "completed_group_1", "mean": 0.05762210115790367, "median": 0.056058838963508606, "p05": 0.03864594176411629, "p95": 0.08028930425643921 }, { "label": "completed_group_2", "mean": 0.045099519193172455, "median": 0.04426075518131256, "p05": 0.03003283590078354, "p95": 0.06244165822863579 }, { "label": "completed_group_3", "mean": 0.056236449629068375, "median": 0.052902162075042725, "p05": 0.03282861411571503, "p95": 0.08620619773864746 }, { "label": "completed_group_4", "mean": 0.029373466968536377, "median": 0.02887408435344696, "p05": 0.020401213318109512, "p95": 0.04018516466021538 }, { "label": "completed_group_5", "mean": 0.07237439602613449, "median": 0.06993190199136734, "p05": 0.04555661231279373, "p95": 0.10748817771673203 }, { "label": "completed_group_6", "mean": 0.31761035323143005, "median": 0.3173876404762268, "p05": 0.16091161966323853, "p95": 0.4724525213241577 }, { "label": "current_group_partial", "mean": 0.3292362689971924, "median": 0.3128150701522827, "p05": 0.12605780363082886, "p95": 0.5847846865653992 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8334553241729736, "group": 8, "latest_source_mass": 0.270294725894928, "layer": 29, "max_source_mass": 0.27155569195747375, "mixer_index": 56, "normalized_entropy": 0.8817056346252721, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1157165914773941, "median": 0.113271564245224, "p05": 0.06316040456295013, "p95": 0.17676270008087158 }, { "label": "completed_group_1", "mean": 0.07161639630794525, "median": 0.07118864357471466, "p05": 0.04294680058956146, "p95": 0.102086141705513 }, { "label": "completed_group_2", "mean": 0.06577154994010925, "median": 0.06490545719861984, "p05": 0.04486069828271866, "p95": 0.09022088348865509 }, { "label": "completed_group_3", "mean": 0.07425849884748459, "median": 0.07179903984069824, "p05": 0.045271825045347214, "p95": 0.11117851734161377 }, { "label": "completed_group_4", "mean": 0.04881400242447853, "median": 0.04833249747753143, "p05": 0.0363147035241127, "p95": 0.0629924014210701 }, { "label": "completed_group_5", "mean": 0.08197253197431564, "median": 0.08029600977897644, "p05": 0.05680408701300621, "p95": 0.11223416775465012 }, { "label": "completed_group_6", "mean": 0.27155569195747375, "median": 0.2687028646469116, "p05": 0.18294592201709747, "p95": 0.3740781545639038 }, { "label": "completed_group_7", "mean": 0.270294725894928, "median": 0.2569413185119629, "p05": 0.13936233520507812, "p95": 0.43686625361442566 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6227548122406006, "group": 8, "latest_source_mass": 0.45173269510269165, "layer": 29, "max_source_mass": 0.45173269510269165, "mixer_index": 57, "normalized_entropy": 0.7385475426494315, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0605507455766201, "median": 0.05293343961238861, "p05": 0.02206447720527649, "p95": 0.1254502236843109 }, { "label": "completed_group_1", "mean": 0.040063515305519104, "median": 0.03803131729364395, "p05": 0.022060411050915718, "p95": 0.06445677578449249 }, { "label": "completed_group_2", "mean": 0.032619886100292206, "median": 0.030926726758480072, "p05": 0.020269516855478287, "p95": 0.050711989402770996 }, { "label": "completed_group_3", "mean": 0.03868011012673378, "median": 0.0344192236661911, "p05": 0.020861119031906128, "p95": 0.06798171997070312 }, { "label": "completed_group_4", "mean": 0.020290488377213478, "median": 0.01974661275744438, "p05": 0.014590941369533539, "p95": 0.027847949415445328 }, { "label": "completed_group_5", "mean": 0.04760073497891426, "median": 0.044880062341690063, "p05": 0.027415042743086815, "p95": 0.07675685733556747 }, { "label": "completed_group_6", "mean": 0.19551873207092285, "median": 0.18908299505710602, "p05": 0.10506269335746765, "p95": 0.30520933866500854 }, { "label": "completed_group_7", "mean": 0.11294309794902802, "median": 0.09412655979394913, "p05": 0.03902517631649971, "p95": 0.24146248400211334 }, { "label": "current_group_partial", "mean": 0.45173269510269165, "median": 0.4618796110153198, "p05": 0.24265648424625397, "p95": 0.6204620599746704 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9787883758544922, "group": 8, "latest_source_mass": 0.1984800100326538, "layer": 30, "max_source_mass": 0.24502354860305786, "mixer_index": 58, "normalized_entropy": 0.9005854004479842, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0788881778717041, "median": 0.055554650723934174, "p05": 0.03367501497268677, "p95": 0.21530622243881226 }, { "label": "completed_group_1", "mean": 0.08108963817358017, "median": 0.07848536968231201, "p05": 0.05139587074518204, "p95": 0.12043403089046478 }, { "label": "completed_group_2", "mean": 0.057054273784160614, "median": 0.056182146072387695, "p05": 0.04127398133277893, "p95": 0.07570968568325043 }, { "label": "completed_group_3", "mean": 0.08211587369441986, "median": 0.0802752673625946, "p05": 0.055154237896203995, "p95": 0.113825723528862 }, { "label": "completed_group_4", "mean": 0.026040805503726006, "median": 0.025504354387521744, "p05": 0.01917124353349209, "p95": 0.03502596169710159 }, { "label": "completed_group_5", "mean": 0.10537257045507431, "median": 0.10299044847488403, "p05": 0.07248030602931976, "p95": 0.14682361483573914 }, { "label": "completed_group_6", "mean": 0.24502354860305786, "median": 0.24406442046165466, "p05": 0.1495005488395691, "p95": 0.3413054645061493 }, { "label": "completed_group_7", "mean": 0.12593510746955872, "median": 0.11572076380252838, "p05": 0.051278628408908844, "p95": 0.2299422323703766 }, { "label": "current_group_partial", "mean": 0.1984800100326538, "median": 0.19106052815914154, "p05": 0.11614266037940979, "p95": 0.3049595355987549 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7590525150299072, "group": 8, "latest_source_mass": 0.3258843719959259, "layer": 30, "max_source_mass": 0.3258843719959259, "mixer_index": 59, "normalized_entropy": 0.800579300438408, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07131639868021011, "median": 0.052984386682510376, "p05": 0.03260878100991249, "p95": 0.19837424159049988 }, { "label": "completed_group_1", "mean": 0.044687941670417786, "median": 0.04335416108369827, "p05": 0.027018578723073006, "p95": 0.06556333601474762 }, { "label": "completed_group_2", "mean": 0.033290669322013855, "median": 0.03222261369228363, "p05": 0.01970716379582882, "p95": 0.05042369291186333 }, { "label": "completed_group_3", "mean": 0.043597716838121414, "median": 0.03997292369604111, "p05": 0.024005703628063202, "p95": 0.07244924455881119 }, { "label": "completed_group_4", "mean": 0.02658817172050476, "median": 0.02605675905942917, "p05": 0.017710059881210327, "p95": 0.03680335357785225 }, { "label": "completed_group_5", "mean": 0.06922741234302521, "median": 0.06634850800037384, "p05": 0.03838749974966049, "p95": 0.10990548133850098 }, { "label": "completed_group_6", "mean": 0.23554196953773499, "median": 0.2292930632829666, "p05": 0.10460641980171204, "p95": 0.3925905227661133 }, { "label": "completed_group_7", "mean": 0.14986534416675568, "median": 0.12543796002864838, "p05": 0.04750998318195343, "p95": 0.3270293176174164 }, { "label": "current_group_partial", "mean": 0.3258843719959259, "median": 0.31322309374809265, "p05": 0.1389845609664917, "p95": 0.5583901405334473 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9736928939819336, "group": 8, "latest_source_mass": 0.20976582169532776, "layer": 31, "max_source_mass": 0.22560220956802368, "mixer_index": 60, "normalized_entropy": 0.8982663467084998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09849467873573303, "median": 0.09779177606105804, "p05": 0.017307927832007408, "p95": 0.19556622207164764 }, { "label": "completed_group_1", "mean": 0.068942591547966, "median": 0.0661388635635376, "p05": 0.044743385165929794, "p95": 0.10368303209543228 }, { "label": "completed_group_2", "mean": 0.0599900484085083, "median": 0.05786054581403732, "p05": 0.040265996009111404, "p95": 0.08643331378698349 }, { "label": "completed_group_3", "mean": 0.06335455179214478, "median": 0.05964011698961258, "p05": 0.04099462926387787, "p95": 0.09641734510660172 }, { "label": "completed_group_4", "mean": 0.04671281576156616, "median": 0.04506188631057739, "p05": 0.03403949737548828, "p95": 0.06450185924768448 }, { "label": "completed_group_5", "mean": 0.0713595524430275, "median": 0.06962447613477707, "p05": 0.048271458595991135, "p95": 0.10105480253696442 }, { "label": "completed_group_6", "mean": 0.22560220956802368, "median": 0.22400957345962524, "p05": 0.13957585394382477, "p95": 0.31992682814598083 }, { "label": "completed_group_7", "mean": 0.1557777225971222, "median": 0.1399117112159729, "p05": 0.06281191110610962, "p95": 0.2967374622821808 }, { "label": "current_group_partial", "mean": 0.20976582169532776, "median": 0.2105630338191986, "p05": 0.08139262348413467, "p95": 0.34620192646980286 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.809161901473999, "group": 8, "latest_source_mass": 0.288146048784256, "layer": 31, "max_source_mass": 0.288146048784256, "mixer_index": 61, "normalized_entropy": 0.8233850650202157, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08854076266288757, "median": 0.08406365662813187, "p05": 0.020090684294700623, "p95": 0.1807844340801239 }, { "label": "completed_group_1", "mean": 0.05356752499938011, "median": 0.05036265030503273, "p05": 0.031541675329208374, "p95": 0.08521421998739243 }, { "label": "completed_group_2", "mean": 0.04243364930152893, "median": 0.03939494490623474, "p05": 0.023794323205947876, "p95": 0.0721740797162056 }, { "label": "completed_group_3", "mean": 0.04702243581414223, "median": 0.043501369655132294, "p05": 0.02430840954184532, "p95": 0.0788305327296257 }, { "label": "completed_group_4", "mean": 0.02648795023560524, "median": 0.025087328627705574, "p05": 0.01698113977909088, "p95": 0.040197182446718216 }, { "label": "completed_group_5", "mean": 0.05820769816637039, "median": 0.05441046878695488, "p05": 0.02834305912256241, "p95": 0.10125020891427994 }, { "label": "completed_group_6", "mean": 0.18489578366279602, "median": 0.17386877536773682, "p05": 0.07876831293106079, "p95": 0.32456493377685547 }, { "label": "completed_group_7", "mean": 0.21069815754890442, "median": 0.19004324078559875, "p05": 0.07121473550796509, "p95": 0.42417171597480774 }, { "label": "current_group_partial", "mean": 0.288146048784256, "median": 0.28367581963539124, "p05": 0.0962119847536087, "p95": 0.5029257535934448 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.8544633388519287, "group": 8, "latest_source_mass": 0.12106549739837646, "layer": 32, "max_source_mass": 0.3525030016899109, "mixer_index": 62, "normalized_entropy": 0.8440026376822011, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.07977832853794098, "median": 0.053830958902835846, "p05": 0.02079753391444683, "p95": 0.2620829939842224 }, { "label": "completed_group_1", "mean": 0.05858254432678223, "median": 0.056158941239118576, "p05": 0.03545547276735306, "p95": 0.09179478138685226 }, { "label": "completed_group_2", "mean": 0.053744811564683914, "median": 0.05185101926326752, "p05": 0.03345105051994324, "p95": 0.08065605163574219 }, { "label": "completed_group_3", "mean": 0.07626347988843918, "median": 0.07124648243188858, "p05": 0.043040163815021515, "p95": 0.12173676490783691 }, { "label": "completed_group_4", "mean": 0.031090255826711655, "median": 0.02958102896809578, "p05": 0.01873353309929371, "p95": 0.04807420074939728 }, { "label": "completed_group_5", "mean": 0.0994817242026329, "median": 0.0973888486623764, "p05": 0.0575672909617424, "p95": 0.14856761693954468 }, { "label": "completed_group_6", "mean": 0.3525030016899109, "median": 0.3572760224342346, "p05": 0.18325307965278625, "p95": 0.5024369955062866 }, { "label": "completed_group_7", "mean": 0.1274903416633606, "median": 0.1073581799864769, "p05": 0.039848171174526215, "p95": 0.2741084098815918 }, { "label": "current_group_partial", "mean": 0.12106549739837646, "median": 0.10882791876792908, "p05": 0.0503288172185421, "p95": 0.22616314888000488 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9456019401550293, "group": 8, "latest_source_mass": 0.18013209104537964, "layer": 32, "max_source_mass": 0.2524128556251526, "mixer_index": 63, "normalized_entropy": 0.885481602665194, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08823961019515991, "median": 0.07105386257171631, "p05": 0.04482517018914223, "p95": 0.1774362027645111 }, { "label": "completed_group_1", "mean": 0.06737187504768372, "median": 0.06628458946943283, "p05": 0.04582075774669647, "p95": 0.09290395677089691 }, { "label": "completed_group_2", "mean": 0.05197303742170334, "median": 0.051264435052871704, "p05": 0.035227708518505096, "p95": 0.07133325934410095 }, { "label": "completed_group_3", "mean": 0.0631948709487915, "median": 0.06016676500439644, "p05": 0.03969947248697281, "p95": 0.09776238352060318 }, { "label": "completed_group_4", "mean": 0.02739548496901989, "median": 0.026907969266176224, "p05": 0.020028207451105118, "p95": 0.036658842116594315 }, { "label": "completed_group_5", "mean": 0.08291903138160706, "median": 0.08066994696855545, "p05": 0.05307728424668312, "p95": 0.12080682814121246 }, { "label": "completed_group_6", "mean": 0.2524128556251526, "median": 0.25180506706237793, "p05": 0.14340050518512726, "p95": 0.36010295152664185 }, { "label": "completed_group_7", "mean": 0.186361163854599, "median": 0.17697101831436157, "p05": 0.08843781799077988, "p95": 0.3156980276107788 }, { "label": "current_group_partial", "mean": 0.18013209104537964, "median": 0.16958200931549072, "p05": 0.08391770720481873, "p95": 0.3105517029762268 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.596132755279541, "group": null, "latest_source_mass": 0.29075440764427185, "layer": null, "max_source_mass": 0.3670029938220978, "mixer_index": null, "normalized_entropy": 0.7264313223797062, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09668220579624176, "median": 0.1013428270816803, "p05": 0.02658359333872795, "p95": 0.15407970547676086 }, { "label": "completed_group_1", "mean": 0.037084318697452545, "median": 0.035886794328689575, "p05": 0.02348068356513977, "p95": 0.053765345364809036 }, { "label": "completed_group_2", "mean": 0.03457995876669884, "median": 0.03355112299323082, "p05": 0.019709739834070206, "p95": 0.0531856045126915 }, { "label": "completed_group_3", "mean": 0.026012206450104713, "median": 0.023950476199388504, "p05": 0.013590577989816666, "p95": 0.04525115340948105 }, { "label": "completed_group_4", "mean": 0.0258483849465847, "median": 0.02512776106595993, "p05": 0.016705092042684555, "p95": 0.03687155246734619 }, { "label": "completed_group_5", "mean": 0.037477150559425354, "median": 0.03506678715348244, "p05": 0.02071693353354931, "p95": 0.061963990330696106 }, { "label": "completed_group_6", "mean": 0.0845583900809288, "median": 0.07240844517946243, "p05": 0.03393660485744476, "p95": 0.1794772744178772 }, { "label": "completed_group_7", "mean": 0.3670029938220978, "median": 0.3608853220939636, "p05": 0.1103469580411911, "p95": 0.6465575098991394 }, { "label": "completed_group_8", "mean": 0.29075440764427185, "median": 0.28802984952926636, "p05": 0.039541684091091156, "p95": 0.5658512711524963 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "optimizer_state_before": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.09253507107496262, 0.19514523446559906, 0.07917427271604538, 0.04215213656425476, 0.12552575767040253, 0.05027730390429497, 0.054851483553647995, 0.036243390291929245, 0.031880609691143036, 0.035054463893175125, 0.050445348024368286, 0.05600574240088463, 0.053015924990177155, 0.032114963978528976, 0.048427484929561615, 0.5742782950401306, 0.04075472429394722, 0.03505101054906845, 0.06577442586421967, 0.08553697913885117, 0.0493597537279129, 0.023477228358387947, 0.03512806445360184, 0.030729109421372414, 0.023035654798150063, 0.019568851217627525, 0.02376176416873932, 0.02176014520227909, 0.03424545377492905, 0.021422436460852623, 0.019477492198348045, 0.017491821199655533 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0284768109158904e-05, "first_to_last_ratio": 0.5012331703882884, "imbalance_abs_log_ratio": 0.6906838762102308, "last_quartile_mean": 2.051892954568757e-05, "mean": 1.8776570954059935e-05, "normalized": [ 0.5986193859992756, 0.6558081812372716, 0.5520121545916151, 0.5273260046333214, 0.6138572699796184, 0.5188085183079506, 0.470653216662131, 0.4448732456112806, 0.571885288304143, 0.5260631358207655, 0.4944655013148357, 0.4611973943964093, 0.3985954392877791, 0.37569126542434245, 0.3598786676337444, 0.4301976753792103, 1.0911627323935584, 1.0892352977591147, 1.0129363601224668, 0.9043812768757663, 3.7939202517290083, 3.07802379201043, 2.391002620377405, 1.8970509950687215, 1.8215330675860186, 1.6199762600873888, 1.323734904300092, 0.9702906094541006, 1.1932084612277043, 0.7841779822795031, 0.5845545188198558, 0.4448785253251717 ], "peak_layer": 21, "peak_normalized": 3.7939202517290083, "population_cv": 0.8048941588500481, "rest_mean": 1.3225986583199335e-05, "spike_contrast": 3.6859047338391244, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.8749726556707176e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.124001937569119e-05, 1.2313828847254626e-05, 1.0364895388192963e-05, 9.901374141918495e-06, 1.152613458543783e-05, 9.741444955579937e-06, 8.837253517413046e-06, 8.353194061783142e-06, 1.0738044693425763e-05, 9.877661796053872e-06, 9.284366569772828e-06, 8.659705599711742e-06, 7.4842555477516726e-06, 7.054193702060729e-06, 6.757287337677553e-06, 8.077637176029384e-06, 2.0488294467213564e-05, 2.0452103854040615e-05, 1.9019471437786706e-05, 1.698117921478115e-05, 7.123681280063465e-05, 5.779473212896846e-05, 4.489483035285957e-05, 3.562011261237785e-05, 3.420214488869533e-05, 3.0417599191423506e-05, 2.4855202354956418e-05, 1.8218730474472977e-05, 2.2404363335226662e-05, 1.4724173524882644e-05, 1.0975929399137385e-05, 8.353293196705636e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.085704995304695e-05, "first_to_last_ratio": 0.49797744750683937, "imbalance_abs_log_ratio": 0.6972004891137765, "last_quartile_mean": 8.204598452721257e-05, "mean": 7.523339269255302e-05, "normalized": [ 0.5949378857857258, 0.650065068423859, 0.5492544663710863, 0.5251722584489763, 0.6053425775579288, 0.5124227888322227, 0.46580426292226623, 0.44156615556169815, 0.5688035499575106, 0.5274402954719286, 0.49517820262129336, 0.4612679954527406, 0.39847860202951046, 0.37431010299451944, 0.3578776058560047, 0.4280250030645366, 1.0938473267662936, 1.088989782662427, 1.0233362931504402, 0.9048102634375241, 3.7996932997999537, 3.0906905066941115, 2.4067960338193375, 1.9114675410445925, 1.823315112659501, 1.6164295945753033, 1.3192322668021113, 0.9632233265477733, 1.1927048821627753, 0.7801549776691259, 0.5836505536753793, 0.44571141718154267 ], "peak_layer": 21, "peak_normalized": 3.7996932997999537, "population_cv": 0.808655698028205, "rest_mean": 5.285295608498725e-05, "spike_contrast": 3.710062121371229, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001960877503734082, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 4.475919558899477e-05, 4.890660056844354e-05, 4.1322276956634596e-05, 3.951049075112678e-05, 4.554197585093789e-05, 3.855130489682779e-05, 3.504403503029607e-05, 3.3220519981114194e-05, 4.27930208388716e-05, 3.9681122871115804e-05, 3.7253936170600355e-05, 3.47027562384028e-05, 2.997889714606572e-05, 2.816061896737665e-05, 2.6924346457235515e-05, 3.2201773137785494e-05, 8.229384548030794e-05, 8.192839595722035e-05, 7.698906119912863e-05, 6.80719458614476e-05, 0.0002858638181351125, 0.00023252313258126378, 0.00018107143114320934, 0.00014380618813447654, 0.0001371741818729788, 0.00012160948244854808, 9.925031918101013e-05, 7.246655877679586e-05, 8.973123476607725e-05, 5.869370579603128e-05, 4.391001129988581e-05, 3.3532382076373324e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.036232439801096916, "first_to_last_ratio": 0.49797744231506097, "imbalance_abs_log_ratio": 0.6972004995395066, "last_quartile_mean": 0.07275919895619154, "mean": 0.06671772431582212, "normalized": [ 0.5949378741845537, 0.6500650614940244, 0.5492544412180356, 0.5251722869429579, 0.6053425762620699, 0.5124227830545606, 0.46580425469520476, 0.4415661488209036, 0.5688035576730777, 0.5274403134409381, 0.495178210228144, 0.46126797835287947, 0.3984786148158961, 0.37431010925252656, 0.35787762380853266, 0.428024993742463, 1.0938473087830916, 1.0889897486949978, 1.0233363030489668, 0.9048102958095574, 3.7996934319878424, 3.0906904475080834, 2.406796061359444, 1.9114674273546057, 1.8233150739769923, 1.6164296712797974, 1.3192322675305295, 0.9632232990014142, 1.1927048767720632, 0.7801549942808673, 0.5836505631099284, 0.4457114015150505 ], "peak_layer": 21, "peak_normalized": 3.7996934319878424, "population_cv": 0.8086557050931692, "rest_mean": 0.046870529651641846, "spike_contrast": 3.710062096477808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.17389257550239562, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03969290107488632, 0.04337086156010628, 0.03664500638842583, 0.0350382998585701, 0.04038707911968231, 0.0341876819729805, 0.031077399849891663, 0.02946028858423233, 0.03794927895069122, 0.03518961742520332, 0.03303716331720352, 0.03077474981546402, 0.026585586369037628, 0.02497311867773533, 0.02387678064405918, 0.028556853532791138, 0.07297900319099426, 0.07265491783618927, 0.06827466934919357, 0.060366883873939514, 0.2535068988800049, 0.20620383322238922, 0.1605759561061859, 0.12752875685691833, 0.1216474324464798, 0.1078445091843605, 0.08801617473363876, 0.06426406651735306, 0.07957455515861511, 0.052050165832042694, 0.038939837366342545, 0.029736850410699844 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.7214510705798602e-06, "first_to_last_ratio": 0.31570143383120763, "imbalance_abs_log_ratio": 1.1529583416761842, "last_quartile_mean": 5.452781920212146e-06, "mean": 4.312751560320294e-06, "normalized": [ 0.5189290406874598, 0.4629408602327028, 0.4379369350734736, 0.4223325243325377, 0.3554137514884539, 0.3403071611993441, 0.32823921317250115, 0.32713072242218383, 0.4058203853809248, 0.3945172118813988, 0.39884681315808357, 0.39929763261093665, 0.39159574260871993, 0.3879244716451889, 0.3882468093992228, 0.3995901564951789, 0.788703492444173, 0.816182126959875, 0.8217008297860132, 0.8161870300366048, 3.918660813887919, 3.3485633171709597, 2.7353702011780077, 2.2808471855925627, 2.1314258695348536, 1.89580066182392, 1.5684885519233662, 1.157279421610267, 1.35167586969034, 0.8709255585803598, 0.6496847126841314, 0.4894349253083356 ], "peak_layer": 21, "peak_normalized": 3.918660813887919, "population_cv": 0.9292175371812949, "rest_mean": 2.808900300503107e-06, "spike_contrast": 4.426482620656244, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2433548363333102e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.2380120299203554e-06, 1.996548917304608e-06, 1.8887132000600104e-06, 1.8214152532891603e-06, 1.5328112112911185e-06, 1.4676602404506411e-06, 1.4156141787680099e-06, 1.4108335335549782e-06, 1.7502025002613664e-06, 1.7014547211147146e-06, 1.7201272157763015e-06, 1.7220714880750165e-06, 1.688855149950541e-06, 1.673021870374214e-06, 1.6744120330258738e-06, 1.7233330709132133e-06, 3.401482217668672e-06, 3.5199907415517373e-06, 3.5437915357761085e-06, 3.5200118873035535e-06, 1.6900210539461114e-05, 1.4441521670960356e-05, 1.1796972103184089e-05, 9.836727258516476e-06, 9.192310244543478e-06, 8.176117262337357e-06, 6.764501449652016e-06, 4.991058631276246e-06, 5.8294422160543036e-06, 3.75608556169027e-06, 2.8019287583447294e-06, 2.110811237798771e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.160356504849915e-05, "first_to_last_ratio": 0.6176036367578212, "imbalance_abs_log_ratio": 0.48190839175616645, "last_quartile_mean": 5.117127420817269e-05, "mean": 5.2418822178879054e-05, "normalized": [ 0.666322518769789, 0.714984736429167, 0.6173684650819016, 0.5916946676135548, 0.6555071657760245, 0.5687098563341721, 0.5173612203652723, 0.49129061612094416, 0.619582115965453, 0.5740227294981308, 0.5363326789838647, 0.4988165849512774, 0.45986272133852035, 0.431797325881486, 0.41220412812819657, 0.4796096795486855, 1.1736071025949124, 1.1561730056375625, 1.0641195939880703, 0.948668636490457, 3.882987390089663, 3.0644523599328255, 2.292877320973713, 1.7720443276024598, 1.7296067191133486, 1.4826105850124727, 1.1786273763067439, 0.8692520371850805, 1.00316951798014, 0.6504669042458316, 0.4961137877350426, 0.39975612432523655 ], "peak_layer": 21, "peak_normalized": 3.882987390089663, "population_cv": 0.786177948625012, "rest_mean": 3.738827217603102e-05, "spike_contrast": 3.5728795266419584, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013358379219425843, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.492784162517637e-05, 3.747865775949322e-05, 3.23617277899757e-05, 3.101593756582588e-05, 3.4360913559794426e-05, 2.9811100830556825e-05, 2.7119465812575072e-05, 2.5752875444595702e-05, 3.24777647620067e-05, 3.008959538419731e-05, 2.811392732837703e-05, 2.6147377866436727e-05, 2.410546221653931e-05, 2.2634307242697105e-05, 2.1607254893751815e-05, 2.514057450753171e-05, 6.151910201879218e-05, 6.0605227190535516e-05, 5.577989577432163e-05, 4.9728092562872916e-05, 0.00020354162552393973, 0.00016063498333096504, 0.00012018992856610566, 9.288847650168464e-05, 9.066394704859704e-05, 7.771670061629266e-05, 6.178225885378197e-05, 4.55651679658331e-05, 5.2584964578272775e-05, 3.40967089869082e-05, 2.6005700419773348e-05, 2.0954745195922442e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.434365933368099e-05, "first_to_last_ratio": 0.8577466133816546, "imbalance_abs_log_ratio": 0.15344654550942588, "last_quartile_mean": 2.8380944854689005e-05, "mean": 3.523111084291486e-05, "normalized": [ 0.7673952990085477, 0.811220254411165, 0.7193610364184987, 0.6887351358833893, 0.7328492204226176, 0.6490237652719044, 0.5963486702580318, 0.5628308587943773, 0.6991412032912354, 0.6479259522662333, 0.6038428974789878, 0.5611268039454765, 0.5540897621717866, 0.518717032406164, 0.49549402526296327, 0.5544919614528937, 1.3117045453116274, 1.2768062454810614, 1.1497523702928032, 1.0122574224190877, 4.0186385000439255, 3.041265540561009, 2.0735563094276896, 1.5089063120619997, 1.5078974578960394, 1.239262930352535, 0.9604083073922689, 0.7448019221905203, 0.7558275520339612, 0.5149618128538204, 0.3996625343446833, 0.32169635859269563 ], "peak_layer": 21, "peak_normalized": 4.0186385000439255, "population_cv": 0.7633311043023641, "rest_mean": 2.5901046110784927e-05, "spike_contrast": 3.30540550486754, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.561346039641649e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.7036188839701936e-05, 2.8580190701177344e-05, 2.534388841013424e-05, 2.4264903913717717e-05, 2.5819092115852982e-05, 2.286582821398042e-05, 2.1010026102885604e-05, 1.982915637199767e-05, 2.4631521228002384e-05, 2.2827151042292826e-05, 2.1274056052789092e-05, 1.976912062673364e-05, 1.952119782799855e-05, 1.8274977264809422e-05, 1.7456804926041514e-05, 1.953536775545217e-05, 4.621280822902918e-05, 4.4983302359469235e-05, 4.0507053199689835e-05, 3.5662953450810164e-05, 0.00014158109843265265, 0.00010714716336224228, 7.30536921764724e-05, 5.316044553183019e-05, 5.3124902478884906e-05, 4.366060966276564e-05, 3.383625153219327e-05, 2.624019907671027e-05, 2.6628644263837487e-05, 1.814267670852132e-05, 1.4080555047257803e-05, 1.1333720067341346e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.111996157822432e-05, "first_to_last_ratio": 0.45955600820785986, "imbalance_abs_log_ratio": 0.777494455152158, "last_quartile_mean": 0.0001765181177688646, "mean": 0.00015741029130822426, "normalized": [ 0.5705309506751165, 0.6126723076713019, 0.5240846716131147, 0.5073054874330758, 0.5540338647553013, 0.48682190889438537, 0.44234061084833826, 0.4249374177361315, 0.550364784679922, 0.5081370833366782, 0.48009264279277103, 0.45140806153022744, 0.3632347905144417, 0.3493062409166391, 0.33465617521315694, 0.40047146041522963, 1.021224360197918, 1.022384831920215, 0.9906834216965009, 0.8745459299517545, 3.8481386526758086, 3.158088682741086, 2.5497611412614916, 2.003665132352204, 1.879007746207825, 1.6962363619877694, 1.389737503600916, 1.0270562090093074, 1.1914290365360227, 0.7723867467392906, 0.5792090660062168, 0.43604671808984163 ], "peak_layer": 21, "peak_normalized": 3.8481386526758086, "population_cv": 0.8430355170943084, "rest_mean": 0.00010821280456083413, "spike_contrast": 3.9096733650063507, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000423076719744131, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 8.980744314612821e-05, 9.644092642702162e-05, 8.249632082879543e-05, 7.985510455910116e-05, 8.72106320457533e-05, 7.663077849429101e-05, 6.962896441109478e-05, 6.688952271360904e-05, 8.663308108225465e-05, 7.998600631253794e-05, 7.557152275694534e-05, 7.105627446435392e-05, 5.717689418816008e-05, 5.498439713846892e-05, 5.267832602839917e-05, 6.30383292445913e-05, 0.00016075122402980924, 0.00016093389422167093, 0.0001559437660034746, 0.00013766252959612757, 0.0006057366263121367, 0.0004971156595274806, 0.00040135864401236176, 0.0003153975121676922, 0.00029577515670098364, 0.0002670050598680973, 0.00021875898528378457, 0.00016166921705007553, 0.0001875431917142123, 0.00012158162280684337, 9.117346780840307e-05, 6.863824091851711e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.116500623524189, 0.0468219518661499, 0.09375449270009995, 0.041188232600688934, 0.024237047880887985, 0.022429833188652992, 0.03027583472430706, 0.04546776041388512, 0.030174262821674347, 0.03663193807005882, 0.05748353153467178, 0.10596779733896255, 0.05394246056675911, 0.06537146866321564, 0.08060523867607117, 0.06690707802772522, 0.033449217677116394, 0.022565271705389023, 0.036697570234537125, 0.07367803901433945, 0.042231619358062744, 0.052135858684778214, 0.06269875913858414, 0.0728035643696785, 0.051585447043180466, 0.04862172156572342, 0.05972515419125557, 0.06787826865911484, 0.06439156830310822, 0.07232685387134552, 0.09627991914749146, 0.1271906942129135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0227459597444977e-05, "first_to_last_ratio": 0.6251750002011836, "imbalance_abs_log_ratio": 0.4697236681166162, "last_quartile_mean": 1.635935473132122e-05, "mean": 1.6436911991490888e-05, "normalized": [ 0.6817536191781184, 0.7484899831245089, 0.6296147869866243, 0.6007633380256553, 0.7005091083036028, 0.5900178858271928, 0.5278659818185938, 0.49878628647857515, 0.6254280014067066, 0.5828037509404734, 0.5411654694992786, 0.5069220940770104, 0.4384272646939669, 0.410733592032448, 0.41012977661150796, 0.4993830470114355, 1.2341367480513281, 1.240449185242777, 1.1141663566211946, 1.0232697399583384, 3.628191161581746, 2.8189670579566104, 2.212584875069539, 1.7731887390792915, 1.8705442026109216, 1.5370133298176474, 1.119075341410653, 0.8897518054493918, 0.8979584939233574, 0.6704127897581839, 0.5087700320942508, 0.46872615535907086 ], "peak_layer": 21, "peak_normalized": 3.628191161581746, "population_cv": 0.7362867375552162, "rest_mean": 1.1990741886283675e-05, "spike_contrast": 3.3731216085867595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.044623055960983e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1205924238311127e-05, 1.2302863979130052e-05, 1.0348922842240427e-05, 9.874694114841986e-06, 1.1514206562424079e-05, 9.698072062747087e-06, 8.676486686454155e-06, 8.198506293410901e-06, 1.0280105016136076e-05, 9.579493962519336e-06, 8.895089194993488e-06, 8.332233846886083e-06, 7.206390364444815e-06, 6.751191904186271e-06, 6.741267043253174e-06, 8.208315193769522e-06, 2.0285397113184445e-05, 2.0389154087752104e-05, 1.8313454347662628e-05, 1.6819394659250975e-05, 5.963625881122425e-05, 4.6335113438544795e-05, 3.636806286522187e-05, 2.9145747248549014e-05, 3.074597043450922e-05, 2.5263752831961028e-05, 1.839414289861452e-05, 1.4624772120441776e-05, 1.475966473662993e-05, 1.1019516023225151e-05, 8.362608241441194e-06, 7.704410563746933e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.0602649278298486e-05, "first_to_last_ratio": 0.6220533452330901, "imbalance_abs_log_ratio": 0.4747294258801449, "last_quartile_mean": 6.527197319883271e-05, "mean": 6.584073776139121e-05, "normalized": [ 0.6772234395168817, 0.7419070719489791, 0.6265330495692852, 0.5983454392285444, 0.6910105284953505, 0.5826444544736231, 0.5218070132506913, 0.49396695197265295, 0.6208398201935033, 0.583554602034964, 0.5417867192724627, 0.5073857163005694, 0.43782857851808293, 0.40867327334274134, 0.40807683163251224, 0.49794392992767184, 1.2379798176145815, 1.2443074207246856, 1.1312899909561613, 1.0270568775609965, 3.6326519816528213, 2.8432167741930536, 2.2290444258353594, 1.784033212733279, 1.868354361784077, 1.5302682478219727, 1.1091440983756962, 0.8793208988009685, 0.8936974950846793, 0.6693107032572058, 0.5098445844785217, 0.4709516894474245 ], "peak_layer": 21, "peak_normalized": 3.6326519816528213, "population_cv": 0.7406565006814017, "rest_mean": 4.789962258655578e-05, "spike_contrast": 3.3971616250516083, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016272275970550255, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.458889088709839e-05, 4.8847708967514336e-05, 4.125139821553603e-05, 3.939550515497103e-05, 4.549664299702272e-05, 3.836174073512666e-05, 3.435615872149356e-05, 3.252314854762517e-05, 4.087655179318972e-05, 3.8421665522037074e-05, 3.567163730622269e-05, 3.340664989082143e-05, 2.8826956622651778e-05, 2.6907349820248783e-05, 2.686807965801563e-05, 3.27849957102444e-05, 8.150950452545658e-05, 8.19261185824871e-05, 7.448496762663126e-05, 6.762218254152685e-05, 0.00023917648650240153, 0.0001871994900284335, 0.00014676192949991673, 0.00011746206291718408, 0.00012301382957957685, 0.00010075399040943012, 7.30268657207489e-05, 5.7895136706065387e-05, 5.884170241188258e-05, 4.406791049405001e-05, 3.3568543585715815e-05, 3.100780668319203e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.036006835056468844, "first_to_last_ratio": 0.6220533380595731, "imbalance_abs_log_ratio": 0.47472943741214146, "last_quartile_mean": 0.05788383865728974, "mean": 0.05838822451187298, "normalized": [ 0.6772234369424218, 0.741907072488623, 0.6265330654654753, 0.5983454331639448, 0.6910105564841633, 0.582644442563147, 0.5218069879368077, 0.4939669174779353, 0.620839814266596, 0.5835546430674807, 0.541786737574626, 0.5073857075992703, 0.43782855000599696, 0.4086732953261738, 0.40807680965990323, 0.4979439245673007, 1.2379798733919256, 1.2443073810897243, 1.1312900315759395, 1.0270568889341698, 3.632651939357648, 2.8432167817930876, 2.229044412718128, 1.7840331838058763, 1.8683543984231819, 1.5302683134059842, 1.1091440736440559, 0.8793209345809946, 0.893697457757885, 0.6693107027108595, 0.5098445760442807, 0.4709516561763931 ], "peak_layer": 21, "peak_normalized": 3.632651939357648, "population_cv": 0.7406564998970533, "rest_mean": 0.04247786428917338, "spike_contrast": 3.3971616070921584, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14430416971445084, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.03954187408089638, 0.04331863671541214, 0.03658215329051018, 0.0349363274872303, 0.04034687951207161, 0.03401957452297211, 0.030467383563518524, 0.02884185127913952, 0.036249734461307526, 0.034072719514369965, 0.03163396567106247, 0.02962535060942173, 0.025564031675457954, 0.023861708119511604, 0.023826880380511284, 0.029074061661958694, 0.07228344678878784, 0.0726528987288475, 0.06605401635169983, 0.05996802821755409, 0.21210409700870514, 0.16601037979125977, 0.1301499456167221, 0.10416653007268906, 0.10908989608287811, 0.0893496498465538, 0.0647609531879425, 0.051341988146305084, 0.05218140780925751, 0.03907986357808113, 0.029768919572234154, 0.02749803103506565 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.7200502355763092e-06, "first_to_last_ratio": 0.39238338914104137, "imbalance_abs_log_ratio": 0.9355158836317786, "last_quartile_mean": 4.383595950230301e-06, "mean": 3.6741805224949076e-06, "normalized": [ 0.6063468981334846, 0.5443420367186595, 0.5139094665495357, 0.4967956421023835, 0.4188390484506744, 0.4022862351030342, 0.3820107165872017, 0.3806321225478404, 0.45980638040996796, 0.4539689687950794, 0.4522322819061978, 0.45153818886200525, 0.44692886894308675, 0.4403179668717643, 0.4568379816001622, 0.47173827055358036, 0.9154474599445339, 0.922288261771489, 0.8927585509522445, 0.8903920996318164, 3.60619457117351, 3.0673607840777577, 2.5879499263363996, 2.1944271419318873, 2.2559707186149534, 1.8903392611722498, 1.375705409268863, 1.1270630373807071, 1.036432964427479, 0.7543267801682417, 0.5745703581277657, 0.5302416008854436 ], "peak_layer": 21, "peak_normalized": 3.60619457117351, "population_cv": 0.8429606581046323, "rest_mean": 2.488658121061841e-06, "spike_contrast": 4.048768854572246, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0076001490233467e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.227827962997253e-06, 2.0000109088869067e-06, 1.8881961523220525e-06, 1.8253168718729285e-06, 1.5388902738777688e-06, 1.4780722494833753e-06, 1.4035763342690188e-06, 1.39851113090117e-06, 1.6894116470211884e-06, 1.6679639429639792e-06, 1.661583041823178e-06, 1.6590328186794068e-06, 1.6420973452113685e-06, 1.6178076975847944e-06, 1.6785052139312029e-06, 1.733251565383398e-06, 3.3635192266956437e-06, 3.3886535675264895e-06, 3.2801560791995144e-06, 3.271461309850565e-06, 1.3249809853732586e-05, 1.1270037248323206e-05, 9.50859521253733e-06, 8.062721462920308e-06, 8.2888436736539e-06, 6.945447694306495e-06, 5.054590019426541e-06, 4.141033059568144e-06, 3.808041810771101e-06, 2.771532763290452e-06, 2.1110752186359605e-06, 1.9482033621898154e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.143130106764147e-05, "first_to_last_ratio": 0.7585265502107228, "imbalance_abs_log_ratio": 0.27637747723236683, "last_quartile_mean": 4.143731166550424e-05, "mean": 4.658882704688949e-05, "normalized": [ 0.7474526909468017, 0.8034952952675303, 0.6937413721331414, 0.6643883363040349, 0.7364124474321028, 0.6370996067440579, 0.5721892936725778, 0.542446525940414, 0.669091824873211, 0.627725699423567, 0.5799860791244953, 0.5432856872099403, 0.4993479246608397, 0.46546811100121066, 0.46232421378322947, 0.5482234365904111, 1.3092639964182107, 1.3047191024182814, 1.1584436064552923, 1.0651739919162182, 3.6948461619806166, 2.7963169559172143, 2.1158202215330553, 1.6473298337183941, 1.7175540409463472, 1.395874122700735, 1.0016414447628996, 0.8142117122540828, 0.7421371569646497, 0.5662099426381535, 0.45351577301773555, 0.4242633912505488 ], "peak_layer": 21, "peak_normalized": 3.6948461619806166, "population_cv": 0.7186539970377201, "rest_mean": 3.4558785719801325e-05, "spike_contrast": 3.2278637078747128, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011155105021316558, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.482294414425269e-05, 3.7433903344208375e-05, 3.2320596801582724e-05, 3.0953073292039335e-05, 3.430859214859083e-05, 2.9681723390240222e-05, 2.6657628040993586e-05, 2.5271947379224002e-05, 3.11722033075057e-05, 2.92450040433323e-05, 2.7020871129934676e-05, 2.5311042918474413e-05, 2.3264034098247066e-05, 2.1685613319277763e-05, 2.153914283553604e-05, 2.554108687036205e-05, 6.0997073887847364e-05, 6.0785332607338205e-05, 5.397052882472053e-05, 4.9625206884229556e-05, 0.00017213854880537838, 0.0001302771270275116, 9.857358236331493e-05, 7.674716471228749e-05, 8.001882815733552e-05, 6.503213808173314e-05, 4.6665300033055246e-05, 3.793316864175722e-05, 3.457529965089634e-05, 2.6379057089798152e-05, 2.1128767912159674e-05, 1.9765933757298626e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.4235268710981472e-05, "first_to_last_ratio": 1.025722660756393, "imbalance_abs_log_ratio": 0.02539739905431976, "last_quartile_mean": 2.3627506379853003e-05, "mean": 3.1929538977237826e-05, "normalized": [ 0.8468036918455434, 0.8945762958025398, 0.792146970968438, 0.7585176838192365, 0.8072161456725058, 0.7130868417798004, 0.6487939523425942, 0.6110459875951377, 0.7424727464984224, 0.6961736990480094, 0.6430387303321913, 0.5996082358384006, 0.5883303956688428, 0.5485366222262686, 0.5418345777794853, 0.6224185100083337, 1.4328317480472454, 1.4116054930014736, 1.22660698514051, 1.1171064576533825, 3.798916633890286, 2.7250739228804366, 1.8992806880428186, 1.414065294394857, 1.4476049125600117, 1.1698550635675442, 0.8584820800626458, 0.7351807895168184, 0.5755234543551883, 0.4483728349764013, 0.36052571503973785, 0.32436683964489244 ], "peak_layer": 21, "peak_normalized": 3.798916633890286, "population_cv": 0.6978402085797324, "rest_mean": 2.4497121086352746e-05, "spike_contrast": 2.9417577410010094, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.206459558801725e-05, "top_five_layers": [ 21, 22, 23, 25, 17 ], "top_five_spike_overlap": 4 }, "values": [ 2.7038051484851167e-05, 2.856340870494023e-05, 2.5292887585237622e-05, 2.4219119950430468e-05, 2.5774039386305958e-05, 2.276853410876356e-05, 2.0715691789519042e-05, 1.9510416677803732e-05, 2.37068124988582e-05, 2.2228505258681253e-05, 2.0531930204015225e-05, 1.9145214537275024e-05, 1.8785118300002068e-05, 1.7514521459816024e-05, 1.7300528270425275e-05, 1.987353607546538e-05, 4.574965714709833e-05, 4.507191260927357e-05, 3.916499554179609e-05, 3.5668694181367755e-05, 0.00012129765673307702, 8.701035403646529e-05, 6.0643156757578254e-05, 4.515045293373987e-05, 4.6221357479225844e-05, 3.7352932849898934e-05, 2.7410937036620453e-05, 2.3473983674193732e-05, 1.837619856814854e-05, 1.4316337910713628e-05, 1.1511419870657846e-05, 1.0356883649365045e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.03353295850684e-05, "first_to_last_ratio": 0.568590855165461, "imbalance_abs_log_ratio": 0.5645941629662194, "last_quartile_mean": 0.0001412884657838731, "mean": 0.00013750306106885546, "normalized": [ 0.6520467512598511, 0.6971711299775947, 0.5978339777797664, 0.5761185587459762, 0.6300627892234183, 0.5553891026221839, 0.4941772783126335, 0.4711518985747209, 0.6059991137617907, 0.5690214037338702, 0.5279200902275386, 0.4965685016208935, 0.4023636162938652, 0.37961683338529517, 0.3803019223025787, 0.4631494229257258, 1.1625925589928703, 1.18192702020461, 1.08886530788824, 1.0107340573700256, 3.686196825835599, 2.9476998645270003, 2.331510774085821, 1.8713443744568392, 1.9713065042712608, 1.6029107997806242, 1.1831279762613542, 0.9483310133411668, 0.8858233036848739, 0.6634841961731901, 0.4991942965805259, 0.46605873579829593 ], "peak_layer": 21, "peak_normalized": 3.686196825835599, "population_cv": 0.7760081772891833, "rest_mean": 9.773891576548556e-05, "spike_contrast": 3.6037789344030697, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00035222944570705294, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 8.965842425823212e-05, 9.586316446075216e-05, 8.2204001955688e-05, 7.921806536614895e-05, 8.663556218380108e-05, 7.636770169483498e-05, 6.795088847866282e-05, 6.478482828242704e-05, 8.33267331472598e-05, 7.824218482710421e-05, 7.259062840603292e-05, 6.827968900324777e-05, 5.532622890314087e-05, 5.219847662374377e-05, 5.2292678446974605e-05, 6.368446338456124e-05, 0.0001598600356373936, 0.00016251858323812485, 0.00014972231292631477, 0.0001389790268149227, 0.0005068633472546935, 0.0004053177544847131, 0.00032058986835181713, 0.00025731557980179787, 0.00027106067864224315, 0.0002204051415901631, 0.00016268371837213635, 0.00013039841724094003, 0.00012180341582279652, 9.123110794462264e-05, 6.864074384793639e-05, 6.408450281014666e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.09253507107496262, 0.30484533309936523, 0.33420246839523315, 0.38682517409324646, 0.12552575767040253, 0.14749252796173096, 0.1726136952638626, 0.1892337054014206, 0.031880609691143036, 0.06123264133930206, 0.10403046011924744, 0.15778414905071259, 0.053015924990177155, 0.09510950744152069, 0.1597633957862854, 0.7120755910873413, 0.04075472429394722, 0.07218439877033234, 0.11449843645095825, 0.16539782285690308, 0.0493597537279129, 0.07179203629493713, 0.10295005142688751, 0.1323672980070114, 0.023035654798150063, 0.06294231116771698, 0.09175154566764832, 0.1252463310956955, 0.03424545377492905, 0.07851024717092514, 0.12469635903835297, 0.19299884140491486 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0284754921485728e-05, "first_to_last_ratio": 0.5012334330235659, "imbalance_abs_log_ratio": 0.690683352232122, "last_quartile_mean": 2.0518892483778473e-05, "mean": 1.877660952231963e-05, "normalized": [ 0.5986256157968327, 0.6557756887645085, 0.552024728579064, 0.5273299589895433, 0.6138703950605617, 0.5187982010550154, 0.47064556551807535, 0.44487320369305056, 0.571873215148553, 0.5260659787052139, 0.4944689419167516, 0.46120603772309077, 0.39860670574115153, 0.37568940388544003, 0.359876426854885, 0.4301817276210402, 1.0911581660743186, 1.0892455573175825, 1.0129382513783356, 0.9043825192325898, 3.793965158948972, 3.077978138202815, 2.391038396737279, 1.8970614359566813, 1.8215147947572354, 1.6199717700575011, 1.323724822749107, 0.9702855164110245, 1.1932092071917983, 0.7841724480827788, 0.5845704650360021, 0.4448715568132013 ], "peak_layer": 21, "peak_normalized": 3.793965158948972, "population_cv": 0.8048970821977687, "rest_mean": 1.322599483602163e-05, "spike_contrast": 3.6859177273800294, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.874992882832885e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.1240159437875263e-05, 1.2313244042161386e-05, 1.0365152775193565e-05, 9.90146872936748e-06, 1.1526404705364257e-05, 9.741271242091898e-06, 8.837128007144202e-06, 8.353210432687774e-06, 1.0737840057117864e-05, 9.877735465124715e-06, 9.284450243285391e-06, 8.659885679662693e-06, 7.484482466679765e-06, 7.054173238429939e-06, 6.757259143341798e-06, 8.077354323177133e-06, 2.048825081146788e-05, 2.0452338503673673e-05, 1.901954601635225e-05, 1.6981237422442064e-05, 7.123780233087018e-05, 5.779399361927062e-05, 4.489559432840906e-05, 3.562038182280958e-05, 3.42018720402848e-05, 3.0417577363550663e-05, 2.485506411176175e-05, 1.8218672266812064e-05, 2.240442336187698e-05, 1.47240998558118e-05, 1.0976251360261813e-05, 8.353179509867914e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.08571245316125e-05, "first_to_last_ratio": 0.4979792148814041, "imbalance_abs_log_ratio": 0.6971969400144402, "last_quartile_mean": 8.204584310078644e-05, "mean": 7.52336177356483e-05, "normalized": [ 0.5949478566233565, 0.6500556287691993, 0.5492555313323417, 0.5251723799753355, 0.6053417822936014, 0.5124155017083939, 0.4658024827358601, 0.4415692350980013, 0.568789034243283, 0.5274402651467243, 0.4951821372611509, 0.46126811471013973, 0.3984797553319404, 0.3743113527690246, 0.3578756649483749, 0.4280192256459612, 1.0938415402881307, 1.088997937173476, 1.0233462881438173, 0.9048151004160258, 3.7997527267737836, 3.0906481863038655, 2.406800439856087, 1.911490836810037, 1.823302308579566, 1.6164332700326205, 1.3192176823718622, 0.9632154163000318, 1.192695801919089, 0.7801502262369593, 0.5836600747187168, 0.4457062154832425 ], "peak_layer": 21, "peak_normalized": 3.7997527267737836, "population_cv": 0.8086595618810438, "rest_mean": 5.285302500447465e-05, "spike_contrast": 3.710077492582206, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019608881848398596, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 4.4760079617844895e-05, 4.890603668172844e-05, 4.132248068344779e-05, 3.951061808038503e-05, 4.554205224849284e-05, 3.855087197734974e-05, 3.5044005926465616e-05, 3.322085103718564e-05, 4.279205677448772e-05, 3.968123928643763e-05, 3.725434362422675e-05, 3.470286901574582e-05, 2.9979073588037863e-05, 2.8160797228338197e-05, 2.6924280973616987e-05, 3.2201434805756435e-05, 8.229365630540997e-05, 8.192925452021882e-05, 7.699004345340654e-05, 6.807251338614151e-05, 0.0002858691441360861, 0.00023252064420375973, 0.00018107230425812304, 0.0001438083709217608, 0.00013717362890020013, 0.00012161012273281813, 9.924951882567257e-05, 7.246618042699993e-05, 8.973082003649324e-05, 5.869352389709093e-05, 4.391085894894786e-05, 3.3532091038068756e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.036232506390661, "first_to_last_ratio": 0.49797921643881055, "imbalance_abs_log_ratio": 0.6971969368869876, "last_quartile_mean": 0.07275907346047461, "mean": 0.06671792385168374, "normalized": [ 0.5949478763664731, 0.6500556352385921, 0.5492555345264337, 0.5251723355462972, 0.6053418267327986, 0.512415499380336, 0.4658024986597426, 0.44156921137133487, 0.5687890699892513, 0.5274402435883548, 0.4951821454127685, 0.46126813432265656, 0.39847974028217725, 0.37431136283691135, 0.35787566010736765, 0.42801924671771074, 1.093841580569198, 1.0889979941148478, 1.0233463082409806, 0.90481507183944, 3.799752645338714, 3.090648148898127, 2.4068002538924085, 1.9114909689417183, 1.8233022505104592, 1.6164333240978725, 1.3192177131312344, 0.9632153929742584, 1.192695726061282, 0.7801502600750712, 0.5836600965214254, 0.4457062437137565 ], "peak_layer": 21, "peak_normalized": 3.799752645338714, "population_cv": 0.8086595420067639, "rest_mean": 0.04687059118791863, "spike_contrast": 3.7100773817599753, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.17389352023601531, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.0396936871111393, 0.043370362371206284, 0.03664518892765045, 0.035038407891988754, 0.04038714990019798, 0.03418729826807976, 0.031077375635504723, 0.02946058101952076, 0.037948425859212875, 0.03518971800804138, 0.03303752467036247, 0.030774852260947227, 0.026585740968585014, 0.024973277002573013, 0.023876721039414406, 0.02855655550956726, 0.07297883927822113, 0.07265568524599075, 0.06827554106712341, 0.06036738306283951, 0.25351160764694214, 0.20620162785053253, 0.1605767160654068, 0.12753070890903473, 0.1216469407081604, 0.10784507542848587, 0.08801546692848206, 0.0642637312412262, 0.07957418262958527, 0.05205000564455986, 0.038940589874982834, 0.0297365952283144 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.7214280916277858e-06, "first_to_last_ratio": 0.3156942462365298, "imbalance_abs_log_ratio": 1.1529811089991324, "last_quartile_mean": 5.452833278241087e-06, "mean": 4.3127742159754234e-06, "normalized": [ 0.5189317449354922, 0.46294464940978675, 0.43791502232063706, 0.4223204732934875, 0.3553613513886626, 0.34029330041035655, 0.32825660023687764, 0.3271476671813151, 0.4058018309603618, 0.39452072784651604, 0.398844507071548, 0.3992943751785707, 0.3916207313588865, 0.38791768893123935, 0.3882593735909855, 0.3995747453406645, 0.7887230737056525, 0.8161419891665606, 0.8217969994517562, 0.8161988751020437, 3.9185816028423237, 3.348456311882453, 2.7353965324598755, 2.280942122101189, 2.131436393875469, 1.8958527027667316, 1.5685247034794665, 1.1572372810973706, 1.3516791024443255, 0.870914762401272, 0.6496906841262116, 0.4894220736419112 ], "peak_layer": 21, "peak_normalized": 3.9185816028423237, "population_cv": 0.9292103644815702, "rest_mean": 2.808923749466043e-06, "spike_contrast": 4.426452208782674, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2433566735126077e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.238035449408926e-06, 1.9965757473983103e-06, 1.8886286170527455e-06, 1.8213728480986902e-06, 1.5325932736232062e-06, 1.4676081718789646e-06, 1.415696601725358e-06, 1.4109140238360851e-06, 1.7501316733614658e-06, 1.7014788227243116e-06, 1.7201263062815997e-06, 1.7220664858541568e-06, 1.6889717926460435e-06, 1.673001406743424e-06, 1.6744750155339716e-06, 1.7232756590601639e-06, 3.4015845358226215e-06, 3.519836127452436e-06, 3.5442249100015033e-06, 3.520081463648239e-06, 1.689995769993402e-05, 1.4441136045206804e-05, 1.1797147635661531e-05, 9.837188372330274e-06, 9.19240392249776e-06, 8.176384653779678e-06, 6.764692898286739e-06, 4.990903107682243e-06, 5.8294867812946904e-06, 3.7560587315965677e-06, 2.8019692308589583e-06, 2.1107668999320595e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.160366418342164e-05, "first_to_last_ratio": 0.6176052392742453, "imbalance_abs_log_ratio": 0.48190579702671593, "last_quartile_mean": 5.1171301947761094e-05, "mean": 5.241891676632804e-05, "normalized": [ 0.6663285342326818, 0.7149829604589701, 0.6173687391102264, 0.5916959595973353, 0.6555070239747636, 0.5687064704554682, 0.5173636528095495, 0.4912923321861497, 0.619570657057656, 0.5740235328549215, 0.5363371245546895, 0.4988167952927593, 0.459862273249172, 0.43179654672369233, 0.4122041130466284, 0.4796080159925161, 1.1736023475997315, 1.156174320077284, 1.0641178126387796, 0.9486702559599485, 3.8830086994480117, 3.0644404452878833, 2.2928751268419547, 1.772053067180281, 1.7296002668230133, 1.4826130454616122, 1.1786287196306404, 0.8692460269316107, 1.0031677772103487, 0.6504658693131994, 0.49611695253952276, 0.39975453545899814 ], "peak_layer": 21, "peak_normalized": 3.8830086994480117, "population_cv": 0.7861794547273071, "rest_mean": 3.738832122137494e-05, "spike_contrast": 3.5728839473194793, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013358413270907475, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.492821997497231e-05, 3.747863229364157e-05, 3.2361800549551845e-05, 3.101606125710532e-05, 3.436096812947653e-05, 2.9810977139277384e-05, 2.7119642254547216e-05, 2.5753011868800968e-05, 3.247722270316444e-05, 3.0089691790635698e-05, 2.8114211090723984e-05, 2.614743607409764e-05, 2.4105482225422747e-05, 2.2634307242697105e-05, 2.160729309252929e-05, 2.514053267077543e-05, 6.151896377559751e-05, 6.060540545149706e-05, 5.5779903050279245e-05, 4.972826718585566e-05, 0.00020354310981929302, 0.0001606346486369148, 0.00012019003042951226, 9.288910223403946e-05, 9.06637724256143e-05, 7.771696982672438e-05, 6.178244075272232e-05, 4.5564935135189444e-05, 5.258496821625158e-05, 3.409671626286581e-05, 2.6005913241533563e-05, 2.0954699721187353e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.4343840550500317e-05, "first_to_last_ratio": 0.8576976250683997, "imbalance_abs_log_ratio": 0.1535036599433085, "last_quartile_mean": 2.8382777145452565e-05, "mean": 3.5231219840170525e-05, "normalized": [ 0.7672930723611481, 0.8111997257980034, 0.7192314911950108, 0.6889029711983062, 0.73287297469821, 0.6490049259482068, 0.596465935802746, 0.5628171909857288, 0.6991700183357967, 0.6477070499213824, 0.6038389641268231, 0.5610866035665576, 0.5540522683260976, 0.5186653981035969, 0.4956305510545183, 0.5544873547004332, 1.3116879927303091, 1.2768834577654484, 1.1499858983830977, 1.0122913610994995, 4.018776000950277, 3.0408232653054914, 2.07353399227015, 1.5086765368659305, 1.5087393189985034, 1.2391962109666426, 0.9602587067880377, 0.7447783979976552, 0.7557785917490731, 0.5148267560191684, 0.39972214388707694, 0.3216148721010727 ], "peak_layer": 21, "peak_normalized": 4.018776000950277, "population_cv": 0.7633301669647466, "rest_mean": 2.5900754365095593e-05, "spike_contrast": 3.3055304953184197, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.561573340557516e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.7032670914195478e-05, 2.8579555873875506e-05, 2.5339402782265097e-05, 2.427089202683419e-05, 2.582000888651237e-05, 2.2865235223434865e-05, 2.1014222511439584e-05, 1.982873618544545e-05, 2.4632612621644512e-05, 2.281950946780853e-05, 2.127398329321295e-05, 1.9767765479627997e-05, 1.951993726834189e-05, 1.8273214664077386e-05, 1.7461668903706595e-05, 1.9535265892045572e-05, 4.621236803359352e-05, 4.498616181081161e-05, 4.051540599903092e-05, 3.566425948520191e-05, 0.00014158638077788055, 0.00010713191295508295, 7.30531319277361e-05, 5.315251473803073e-05, 5.3154726629145443e-05, 4.365839413367212e-05, 3.38310856022872e-05, 2.623945147206541e-05, 2.6627001716406085e-05, 1.8137974620913155e-05, 1.4082698726269882e-05, 1.1330884262861218e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.112810064631049e-05, "first_to_last_ratio": 0.4595943192820276, "imbalance_abs_log_ratio": 0.7774110932187634, "last_quartile_mean": 0.00017652111273491755, "mean": 0.00015741467757379723, "normalized": [ 0.5707305382524269, 0.6126189981937518, 0.5242133090337544, 0.5073922534137821, 0.5539956859087919, 0.4868585867686246, 0.4423060989265058, 0.42491050886960396, 0.550246837152428, 0.5080246572887676, 0.4801281677476931, 0.45131986478447, 0.36322233500428036, 0.3493613334959881, 0.33458611504256075, 0.4005281085993613, 1.0209699732329367, 1.022227385527565, 0.9907859769111992, 0.8744049903891328, 3.8474408994912648, 3.15824288565664, 2.5499603975894147, 2.0045124714088103, 1.8789611202618999, 1.6962425295437986, 1.3895520720535863, 1.0272169143608445, 1.191767182381932, 0.7719440072425139, 0.579364871029719, 0.4359629244359521 ], "peak_layer": 21, "peak_normalized": 3.8474408994912648, "population_cv": 0.8430185744802908, "rest_mean": 0.00010821315892691793, "spike_contrast": 3.909902293423383, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00042310287826694546, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 8.984136366052553e-05, 9.64352220762521e-05, 8.251886902144179e-05, 7.987098797457293e-05, 8.720705227460712e-05, 7.663868746021762e-05, 6.962547195143998e-05, 6.688715075142682e-05, 8.661692845635116e-05, 7.997053762665018e-05, 7.557922072010115e-05, 7.104437099769711e-05, 5.717652675230056e-05, 5.4994601669022813e-05, 5.266876542009413e-05, 6.30490030744113e-05, 0.0001607156591489911, 0.00016091359429992735, 0.00015596425510011613, 0.0001376441796310246, 0.0006056436686776578, 0.0004971537855453789, 0.0004014011938124895, 0.00031553968437947333, 0.0002957760589197278, 0.0002670134708750993, 0.00021873589139431715, 0.00016169901937246323, 0.00018760164675768465, 0.00012151531700510532, 9.120053437072784e-05, 6.862696318421513e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.15345558524131775, 0.3249167501926422, 0.3818626403808594, 0.40600064396858215, 0.1351035088300705, 0.15324510633945465, 0.18001793324947357, 0.20391276478767395, 0.04374876990914345, 0.08211024105548859, 0.1380448192358017, 0.2295975536108017, 0.08199761807918549, 0.13892096281051636, 0.21687114238739014, 0.7360114455223083, 0.057135771960020065, 0.08251863718032837, 0.12661415338516235, 0.19619159400463104, 0.06753352284431458, 0.095451220870018, 0.1274663656949997, 0.15628865361213684, 0.058436937630176544, 0.08794140070676804, 0.12247690558433533, 0.1640285700559616, 0.07537907361984253, 0.12290703505277634, 0.19157102704048157, 0.29249706864356995 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0227535881313088e-05, "first_to_last_ratio": 0.6251784250073547, "imbalance_abs_log_ratio": 0.4697181899756329, "last_quartile_mean": 1.635938713206997e-05, "mean": 1.643692004904551e-05, "normalized": [ 0.681769220714557, 0.748500240032939, 0.6296107710699952, 0.6007700154109977, 0.7004931611617099, 0.590031263703908, 0.5278674383583044, 0.49879356717844686, 0.6254239322100563, 0.5827967700199829, 0.5411623822603087, 0.5069232288890692, 0.43843496230947776, 0.4107323947027216, 0.41012888390613456, 0.49939331536990256, 1.2341393523450959, 1.24043551870817, 1.1141674704177404, 1.0232453347314594, 3.6281949162463376, 2.81895018298875, 2.2126026034623427, 1.7731690568174834, 1.8705264645918427, 1.5370337133448664, 1.1190747928279094, 0.8897567918614906, 0.8979560617670435, 0.6704137337604921, 0.5087765332459856, 0.46872592558447873 ], "peak_layer": 21, "peak_normalized": 3.6281949162463376, "population_cv": 0.736284928876382, "rest_mean": 1.19907677395127e-05, "spike_contrast": 3.373106993578246, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.04461425205227e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1206186172785237e-05, 1.2303038602112792e-05, 1.0348861906095408e-05, 9.87480871117441e-06, 1.1513950084918179e-05, 9.698296707938425e-06, 8.67651488078991e-06, 8.198629984690342e-06, 1.0280043170496356e-05, 9.579383913660422e-06, 8.895042810763698e-06, 8.332256584253628e-06, 7.206520422187168e-06, 6.751175533281639e-06, 6.741255674569402e-06, 8.208487997762859e-06, 2.0285449863877147e-05, 2.0388939447002485e-05, 1.831348163250368e-05, 1.681900175753981e-05, 5.963634976069443e-05, 4.63348587800283e-05, 3.6368372093420476e-05, 2.914543802035041e-05, 3.074569394811988e-05, 2.5264100258937106e-05, 1.839414289861452e-05, 1.462486125092255e-05, 1.4759631994820666e-05, 1.1019536941603292e-05, 8.36271919979481e-06, 7.704410563746933e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.0602767057862366e-05, "first_to_last_ratio": 0.6220527400735582, "imbalance_abs_log_ratio": 0.47473039872247574, "last_quartile_mean": 6.52722260383598e-05, "mean": 6.584079096683126e-05, "normalized": [ 0.6772375898692375, 0.7419156446131505, 0.6265276809050203, 0.5983548462080928, 0.6909826192767372, 0.5826528795658069, 0.521809962087311, 0.49396705008933894, 0.6208428547650034, 0.583552196572707, 0.5417821373953783, 0.5073870191655783, 0.43782941267698894, 0.4086714236070873, 0.40807611508949443, 0.4979465665227966, 1.2379953934660572, 1.244289507430791, 1.1312943811704865, 1.027048864560996, 3.6326329119144347, 2.8432062989929086, 2.2290691465672023, 1.784011106006968, 1.8683393699616757, 1.5302817088353375, 1.1091498325862283, 0.8793222326330687, 0.8937044532253541, 0.6693159640810682, 0.5098515212827301, 0.47095130887496306 ], "peak_layer": 21, "peak_normalized": 3.6326329119144347, "population_cv": 0.7406534740263053, "rest_mean": 4.7899763524549565e-05, "spike_contrast": 3.3971428496041334, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016272233915515244, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.458985858946107e-05, 4.884831287199631e-05, 4.1251078073401004e-05, 3.93961563531775e-05, 4.549484219751321e-05, 3.83623264497146e-05, 3.435638063820079e-05, 3.252318128943443e-05, 4.087678462383337e-05, 3.8421538192778826e-05, 3.567136445781216e-05, 3.340676266816445e-05, 2.882703483919613e-05, 2.6907249775831588e-05, 2.686805419216398e-05, 3.278519579907879e-05, 8.15105959190987e-05, 8.192500536097214e-05, 7.448531687259674e-05, 6.762170960428193e-05, 0.00023917542421258986, 0.00018719895160757005, 0.00014676367572974414, 0.00011746070231311023, 0.00012301294191274792, 0.0001007549581117928, 7.302730227820575e-05, 5.789527131128125e-05, 5.8842208090936765e-05, 4.4068292481824756e-05, 3.356902743689716e-05, 3.100780668319203e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.036006940295919776, "first_to_last_ratio": 0.6220527516263364, "imbalance_abs_log_ratio": 0.47473038015045216, "last_quartile_mean": 0.057884062407538295, "mean": 0.058388272824231535, "normalized": [ 0.6772375510524524, 0.7419156461027798, 0.6265276980973783, 0.5983548273880008, 0.6909826136489176, 0.5826528927482733, 0.5218099376842688, 0.49396708297188446, 0.62084280967513, 0.583552246154482, 0.5417821421507241, 0.5073870104266044, 0.43782939997095355, 0.4086714259278741, 0.4080761210926988, 0.49794657504969064, 1.2379954375746993, 1.244289507773514, 1.1312943272761324, 1.027048893286559, 3.6326331106798357, 2.843206262563162, 2.229069109978408, 1.7840111633852547, 1.868339454059732, 1.5302815940748427, 1.109149791313182, 0.879322184865126, 0.8937044383290575, 0.6693159548860595, 0.5098515233166675, 0.47095126649565455 ], "peak_layer": 21, "peak_normalized": 3.6326331106798357, "population_cv": 0.7406534951631871, "rest_mean": 0.04247798943133266, "spike_contrast": 3.397142969281968, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14430380314588548, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.039542730897665024, 0.04331917315721512, 0.036581870168447495, 0.03493690490722656, 0.04034528136253357, 0.03402009606361389, 0.030467581003904343, 0.028841884806752205, 0.03624993935227394, 0.03407260775566101, 0.03163372352719307, 0.02962545119225979, 0.025564102455973625, 0.02386161871254444, 0.023826859891414642, 0.02907424047589302, 0.07228441536426544, 0.07265191525220871, 0.0660543218255043, 0.059967610985040665, 0.21210317313671112, 0.16600990295410156, 0.13015149533748627, 0.10416533052921295, 0.10908911377191544, 0.08935049921274185, 0.06476134061813354, 0.05134210363030434, 0.05218185856938362, 0.03908020257949829, 0.029769349843263626, 0.02749803103506565 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.7200537598682786e-06, "first_to_last_ratio": 0.39238557455548834, "imbalance_abs_log_ratio": 0.9355103140576796, "last_quartile_mean": 4.3835805172420805e-06, "mean": 3.6742107312193184e-06, "normalized": [ 0.6064111606904664, 0.5442957897497384, 0.513901156952112, 0.49677880949687814, 0.4188469295416997, 0.4022651669595537, 0.3819926927365715, 0.38064734156709856, 0.45982843639728, 0.453964184318293, 0.45225749435035856, 0.4515574352379086, 0.44690260682868216, 0.4403067968449148, 0.4568377219852153, 0.47174268441611755, 0.9154831899852715, 0.9223194180598117, 0.8927546763320205, 0.8904547694224777, 3.606171110034034, 3.0673568527539175, 2.587977908068276, 2.1944076145236275, 2.255921476109835, 1.8902962427769419, 1.3756718203329616, 1.127065033693052, 1.0364317453291698, 0.7543213208282148, 0.5745931723466826, 0.5302372413308171 ], "peak_layer": 21, "peak_normalized": 3.606171110034034, "population_cv": 0.8429541336830753, "rest_mean": 2.4886878606964644e-06, "spike_contrast": 4.048733628339768, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.007603423204273e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.228082394140074e-06, 1.9998574316559825e-06, 1.8881811456594733e-06, 1.8252700328957872e-06, 1.5389318832603749e-06, 1.4780069932385231e-06, 1.4035216509000747e-06, 1.3985785471959389e-06, 1.689506575530686e-06, 1.6679600776114967e-06, 1.6616893390164478e-06, 1.6591171743129962e-06, 1.6420143538198317e-06, 1.6177799579963903e-06, 1.6785180605438654e-06, 1.7332820334559074e-06, 3.3636781608947786e-06, 3.388795903447317e-06, 3.280168812125339e-06, 3.2717184694774915e-06, 1.324983259110013e-05, 1.1270115464867558e-05, 9.508776201982982e-06, 8.062716005952097e-06, 8.28873089631088e-06, 6.945346740394598e-06, 5.054508164903382e-06, 4.141074441577075e-06, 3.8080686408648035e-06, 2.771535491774557e-06, 2.1111763999215327e-06, 1.9482033621898154e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.1431425668415613e-05, "first_to_last_ratio": 0.7585286956104332, "imbalance_abs_log_ratio": 0.27637464885849405, "last_quartile_mean": 4.1437358731855056e-05, "mean": 4.658893448095114e-05, "normalized": [ 0.7474561991339586, 0.8034972686588406, 0.6937382106307133, 0.6643963308091924, 0.7364006760736782, 0.6371054777478163, 0.5721931279302918, 0.5424472272278985, 0.6690921560299369, 0.6277240957148766, 0.5799842731563194, 0.5432849029139358, 0.49934763211908073, 0.46546535876668305, 0.46232326479393904, 0.5482249444672911, 1.3092714408814747, 1.304711330444448, 1.1584451517697443, 1.0651690368492541, 3.6948591936119937, 2.7963192533283774, 2.1158287733747994, 1.6473154151775933, 1.7175425839388379, 1.3958754328475529, 1.0016444448776634, 0.8142087414676477, 0.7421362264642747, 0.5662113699947803, 0.45351804589774375, 0.42426241289936223 ], "peak_layer": 21, "peak_normalized": 3.6948591936119937, "population_cv": 0.7186553093862196, "rest_mean": 3.455886885435407e-05, "spike_contrast": 3.2278628485990217, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011155128886457533, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.482318788883276e-05, 3.743408160516992e-05, 3.232052404200658e-05, 3.09535171254538e-05, 3.430812284932472e-05, 2.968206536024809e-05, 2.6657868147594854e-05, 2.527203832869418e-05, 3.117229061899707e-05, 2.9244996767374687e-05, 2.7020849302061833e-05, 2.5311064746347256e-05, 2.3264074116013944e-05, 2.168553510273341e-05, 2.153914829250425e-05, 2.5541216018609703e-05, 6.0997561377007514e-05, 6.078511069063097e-05, 5.3970725275576115e-05, 4.962509046890773e-05, 0.00017213955288752913, 0.000130277534481138, 9.857420809566975e-05, 7.674666994716972e-05, 8.001847891137004e-05, 6.503234908450395e-05, 4.666554741561413e-05, 3.793311771005392e-05, 3.457533603068441e-05, 2.63791844190564e-05, 2.1128922526258975e-05, 1.9765933757298626e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.4235376486103632e-05, "first_to_last_ratio": 1.0257631927899071, "imbalance_abs_log_ratio": 0.025436913861062768, "last_quartile_mean": 2.3626677830179688e-05, "mean": 3.192918461536465e-05, "normalized": [ 0.8468527407513076, 0.8944882935849241, 0.7921265941283154, 0.7585325396809063, 0.8071309908556205, 0.7131774186125125, 0.6489677316897047, 0.6110056554397294, 0.7423413545118936, 0.6962789572096383, 0.6429117038614185, 0.5998452181649871, 0.5884347987626426, 0.548496621769699, 0.5418484530493665, 0.6223045285782076, 1.432791592163331, 1.4116294770658595, 1.2264918474272424, 1.117234731658279, 3.7992903581080144, 2.725020991299199, 1.8993221620386231, 1.4137054453269882, 1.4476142562135503, 1.1700817965819004, 0.8581975882692241, 0.7352593061462055, 0.5755171944998035, 0.4482357862418949, 0.3604934267212365, 0.32437043958777473 ], "peak_layer": 21, "peak_normalized": 3.7992903581080144, "population_cv": 0.6978802845307617, "rest_mean": 2.4496835302906454e-05, "spike_contrast": 2.941762477134701, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.206387090263888e-05, "top_five_layers": [ 21, 22, 23, 25, 17 ], "top_five_spike_overlap": 4 }, "values": [ 2.7039317501476035e-05, 2.8560281862155534e-05, 2.5291956262663007e-05, 2.421932549623307e-05, 2.57710344158113e-05, 2.2771173462388106e-05, 2.072101051453501e-05, 1.9508912373567e-05, 2.3702354155830108e-05, 2.2231619368540123e-05, 2.0527646483969875e-05, 1.915256871143356e-05, 1.878824332379736e-05, 1.7513049897388555e-05, 1.7300779290962964e-05, 1.9869676179951057e-05, 4.5747867261525244e-05, 4.5072178181726485e-05, 3.9160884625744075e-05, 3.567239400581457e-05, 0.00012130824325140566, 8.700769831193611e-05, 6.064380795578472e-05, 4.5138462155591697e-05, 4.6221142838476226e-05, 3.735975769814104e-05, 2.7401549232308753e-05, 2.347623012610711e-05, 1.837579475250095e-05, 1.4311803170130588e-05, 1.1510261174407788e-05, 1.0356883649365045e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.033554240682861e-05, "first_to_last_ratio": 0.5685808176961833, "imbalance_abs_log_ratio": 0.5646118163593131, "last_quartile_mean": 0.0001412913343301625, "mean": 0.0001375067947719799, "normalized": [ 0.6520214267884701, 0.6973466566797177, 0.5977318663617682, 0.5762119700788582, 0.6301734142605383, 0.5553052347453614, 0.49401564943776766, 0.47103073872558565, 0.6061315575728188, 0.5689597597401734, 0.527894379296066, 0.49653787440978014, 0.4023372137884509, 0.37963867061667483, 0.38027646277267696, 0.46302064917750796, 1.162580251769062, 1.1821961108062278, 1.0889626285084961, 1.0105664982562483, 3.6859858284676985, 2.947740045374168, 2.3319071788655306, 1.871247421579714, 1.9714330950080552, 1.60290198738071, 1.1831372292603408, 0.9483145762048698, 0.8857899382807543, 0.6633972344648077, 0.49916037035168714, 0.4660460809694134 ], "peak_layer": 21, "peak_normalized": 3.6859858284676985, "population_cv": 0.7760232497205517, "rest_mean": 9.774026990204152e-05, "spike_contrast": 3.6038986737265986, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003522460290696472, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 8.965737652033567e-05, 9.588990360498428e-05, 8.219219307648018e-05, 7.923306111479178e-05, 8.665312634548172e-05, 7.635824294993654e-05, 6.793050852138549e-05, 6.47699271212332e-05, 8.334720769198611e-05, 7.823583291610703e-05, 7.258906407514587e-05, 6.827733159298077e-05, 5.5324100685538724e-05, 5.220289676799439e-05, 5.229059752309695e-05, 6.36684853816405e-05, 0.00015986268408596516, 0.00016255999798886478, 0.00014973976067267358, 0.00013895976007916033, 0.0005068480968475342, 0.0004053342854604125, 0.00032065308187156916, 0.0002573092351667583, 0.00027108544600196183, 0.00022040991461835802, 0.00016268940817099065, 0.00013039969780948013, 0.00012180213525425643, 9.122162737185135e-05, 6.863794260425493e-05, 6.408450281014666e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.06279198825359344, 0.07690861076116562, 0.18058517575263977, 0.15882407128810883, 0.19892723858356476, 0.15647411346435547, 0.1533493995666504, 0.1527898609638214, 0.16355258226394653, 0.11789151281118393, 0.11456070095300674, 0.12148918211460114, 0.15094918012619019, 0.09854138642549515, 0.11213528364896774, 0.16254977881908417, 0.2323070764541626, 0.1390199512243271, 0.14812250435352325, 0.15333011746406555, 0.1781008094549179, 0.07593096792697906, 0.09691466391086578, 0.09368696808815002, 0.10724716633558273, 0.06538715958595276, 0.07412076741456985, 0.07655765861272812, 0.08420819044113159, 0.07230556756258011, 0.07723049819469452, 0.08422473073005676 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.327331502438028e-06, "first_to_last_ratio": 0.938169966952749, "imbalance_abs_log_ratio": 0.06382414494874765, "last_quartile_mean": 7.810238827232752e-06, "mean": 1.1439842303673231e-05, "normalized": [ 1.135812197656909, 0.7535691222858811, 0.7074574232777815, 0.383535267321132, 0.46428857856750727, 0.7001642723538085, 0.5761929500129469, 0.40305886578711947, 0.38956488690336305, 0.6013590312167297, 0.5758846795304952, 0.5280893188480897, 0.29011217837515946, 0.35128556371858605, 0.25293197560815023, 1.0607061392911914, 0.7223626091760469, 1.3340410494507338, 2.2261654108539917, 2.4044148372801817, 3.331737764852035, 3.2127220644280428, 2.5083141139795067, 1.6244489369321418, 0.9349893421747716, 1.0552287432476088, 1.0681431895248323, 0.5998971016831224, 0.6978207806681275, 0.5604387184364499, 0.3068330794026248, 0.23842980715493145 ], "peak_layer": 21, "peak_normalized": 3.331737764852035, "population_cv": 0.8302470146151533, "rest_mean": 8.638262114699399e-06, "spike_contrast": 3.075662091674845, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6568375324131922e-05, "top_five_layers": [ 21, 22, 23, 20, 19 ], "top_five_spike_overlap": 3 }, "values": [ 1.2993512427783571e-05, 8.62071192386793e-06, 8.093201358860824e-06, 4.387582976050908e-06, 5.311388122208882e-06, 8.009768862393685e-06, 6.591556484636385e-06, 4.61092986370204e-06, 4.45656087322277e-06, 6.8794524850090966e-06, 6.5880299189302605e-06, 6.041258529876359e-06, 3.3188375709869433e-06, 4.018651452497579e-06, 2.893501914513763e-06, 1.2134310964029282e-05, 8.263714335043915e-06, 1.5261219232343137e-05, 2.5466981242061593e-05, 2.750612657109741e-05, 3.8114554627100006e-05, 3.675303378258832e-05, 2.8694717912003398e-05, 1.8583439668873325e-05, 1.0696130630094558e-05, 1.2071650417055935e-05, 1.2219389645906631e-05, 6.862728241685545e-06, 7.982959687069524e-06, 6.411330559785711e-06, 3.5101220419164747e-06, 2.727599394347635e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.5874984658003086e-05, "first_to_last_ratio": 0.8450240698942337, "imbalance_abs_log_ratio": 0.16839016694872203, "last_quartile_mean": 3.062041139401117e-05, "mean": 4.481995506466774e-05, "normalized": [ 0.6446676986966234, 0.6572038842869348, 0.7204290485695262, 0.39682677548592965, 0.4626069284501252, 0.7305503021209181, 0.5844980906502212, 0.4216930592624511, 0.3971779114095773, 0.5863820168610077, 0.5970881316925298, 0.5421931111926823, 0.2798279164088101, 0.3526838924872086, 0.2505358061975237, 1.06664298163141, 0.7393280508622739, 1.3538526055113962, 2.339426776387917, 2.517206586089556, 3.4365096549417373, 3.2562764392234147, 2.588043510314798, 1.612852686817098, 0.9533189353197188, 1.0548076880199933, 1.0535523324501517, 0.599042999372865, 0.6907876896075521, 0.5683312683754906, 0.3059266864656475, 0.239728534836911 ], "peak_layer": 21, "peak_normalized": 3.4365096549417373, "population_cv": 0.8603945180808654, "rest_mean": 3.3453944423753354e-05, "spike_contrast": 3.174406317546299, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010619641252560542, "top_five_layers": [ 21, 22, 23, 20, 19 ], "top_five_spike_overlap": 3 }, "values": [ 2.8893977287225425e-05, 2.9455848562065512e-05, 3.2289597584167495e-05, 1.778575824573636e-05, 2.0734021745738573e-05, 3.274323171353899e-05, 2.6197178158327006e-05, 1.8900263967225328e-05, 1.780149614205584e-05, 2.6281615646439604e-05, 2.6761463232105598e-05, 2.430107088002842e-05, 1.2541874639282469e-05, 1.5807276213308796e-05, 1.1229003575863317e-05, 4.780689050676301e-05, 3.31366500176955e-05, 6.067961294320412e-05, 0.00010485300299478695, 0.00011282108607701957, 0.0001540242083137855, 0.00014594616368412971, 0.0001159959938377142, 7.228798494907096e-05, 4.272771184332669e-05, 4.727643317892216e-05, 4.722016819869168e-05, 2.6849080313695595e-05, 3.096107320743613e-05, 2.547258191043511e-05, 1.3711620340473019e-05, 1.0744622159108985e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.022946195444092155, "first_to_last_ratio": 0.8450240753189588, "imbalance_abs_log_ratio": 0.16839016052911188, "last_quartile_mean": 0.02715448720846325, "mean": 0.0397467844595667, "normalized": [ 0.6446677141451772, 0.6572038849412665, 0.7204290502088391, 0.39682678147434747, 0.46260695282762065, 0.7305502877937194, 0.584498096406077, 0.4216930673859841, 0.3971779243477385, 0.5863820273620618, 0.5970881594088124, 0.5421931359539319, 0.2798279255524417, 0.3526838858930204, 0.25053580074847415, 1.0666429892428337, 0.7393281098238819, 1.353852553770995, 2.339426761851287, 2.517206560586814, 3.4365097564682574, 3.2562762221327364, 2.5880436017094564, 1.6128525942011334, 0.953318925735159, 1.0548076983098749, 1.053552337934538, 0.5990430406264167, 0.6907876804744102, 0.5683312310796297, 0.30592670279868195, 0.23972853880438227 ], "peak_layer": 21, "peak_normalized": 3.4365097564682574, "population_cv": 0.8603945066377574, "rest_mean": 0.029667292721569538, "spike_contrast": 3.1744062637802095, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09417603984475136, "top_five_layers": [ 21, 22, 23, 20, 19 ], "top_five_spike_overlap": 3 }, "values": [ 0.025623468682169914, 0.02612174116075039, 0.02863473817706108, 0.015772588551044464, 0.018387138843536377, 0.029037024825811386, 0.02323191985487938, 0.016760943457484245, 0.01578654535114765, 0.023306800052523613, 0.02373233437538147, 0.021550433710217476, 0.011122260242700577, 0.014018050394952297, 0.009957992471754551, 0.04239562898874283, 0.029385915026068687, 0.05381128564476967, 0.09298469126224518, 0.10005086660385132, 0.13659021258354187, 0.129426509141922, 0.10286641120910645, 0.06410570442676544, 0.037891361862421036, 0.04192521423101425, 0.04187531769275665, 0.02381003461778164, 0.02745658904314041, 0.022589338943362236, 0.012159602716565132, 0.009528438560664654 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.3183985646824112e-06, "first_to_last_ratio": 0.4757249717009145, "imbalance_abs_log_ratio": 0.7429153822802956, "last_quartile_mean": 4.873401024951818e-06, "mean": 3.6109257539607142e-06, "normalized": [ 2.4889669214180863, 1.2151073868036804, 0.21016319311621645, 0.26737414578311336, 0.229833309489438, 0.21558164250894266, 0.25728077609312594, 0.25210078746297815, 0.3035722943409423, 0.3462435475099886, 0.3415130262480174, 0.41431792433763126, 0.2453034593327441, 0.35760301980181497, 0.36403270780468855, 0.36703166516115565, 0.27418249335708944, 0.6688605260814774, 0.9706460648458304, 1.0693924336470282, 1.9436699367464498, 3.518245935177478, 2.6603939519517374, 2.2215709886204884, 1.317163303854247, 1.97794154021279, 1.9610047181494767, 1.276948756957183, 1.7693726874433637, 1.1962434844187537, 0.705933272147052, 0.592404099176991 ], "peak_layer": 22, "peak_normalized": 3.518245935177478, "population_cv": 0.8711221431824825, "rest_mean": 2.7200910966274493e-06, "spike_contrast": 3.096011348297048, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.421432903560344e-06, "top_five_layers": [ 22, 23, 1, 24, 26 ], "top_five_spike_overlap": 3 }, "values": [ 8.98747475730488e-06, 4.387662556837313e-06, 7.588836865579651e-07, 9.654681889514904e-07, 8.299110163534351e-07, 7.78449305016693e-07, 9.290217803936685e-07, 9.103172260438441e-07, 1.0961770158246509e-06, 1.2502597428465378e-06, 1.2331781817920273e-06, 1.4960712633182993e-06, 8.857725788402604e-07, 1.291277953896497e-06, 1.3144950798960053e-06, 1.3253240922495024e-06, 9.900526265482767e-07, 2.415205699435319e-06, 3.5049308735324303e-06, 3.861496679746779e-06, 7.018447831796948e-06, 1.2704124856099952e-05, 9.606485036783852e-06, 8.021927897061687e-06, 4.756178896059282e-06, 7.142200047383085e-06, 7.081042440404417e-06, 4.610967152984813e-06, 6.3890734054439235e-06, 4.31954640589538e-06, 2.549072632973548e-06, 2.139127218470094e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.7170206092487206e-05, "first_to_last_ratio": 0.8911678909117117, "imbalance_abs_log_ratio": 0.11522243950697675, "last_quartile_mean": 1.9267083416707464e-05, "mean": 2.9415226521223303e-05, "normalized": [ 0.5575790030356597, 0.6724660826854738, 0.7144899488628155, 0.417230472852585, 0.46200781574081473, 0.7501512042139171, 0.6407733041668837, 0.45504858840534107, 0.42645564325893864, 0.6277929374359318, 0.6344293066873767, 0.5088524459644442, 0.30594071619112606, 0.3634377152793242, 0.2738683738112511, 1.0754677959704142, 0.7641943868594835, 1.3657039182645663, 2.452287035990965, 2.1509880308711593, 3.185322927772963, 3.4979418159477316, 2.7764798392086663, 1.681060761721552, 0.9405171981782515, 1.0485813422407628, 1.0601218645686263, 0.5229743179246872, 0.6462290554255541, 0.525396773887863, 0.28719893118189943, 0.20901044539297084 ], "peak_layer": 22, "peak_normalized": 3.4979418159477316, "population_cv": 0.8613910548049479, "rest_mean": 2.1700459607624605e-05, "spike_contrast": 3.2752747702027287, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.107496785465627e-05, "top_five_layers": [ 22, 21, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.6401312677771784e-05, 1.9780742150032893e-05, 2.1016883692936972e-05, 1.2272928870515898e-05, 1.3590064554591663e-05, 2.2065867597120814e-05, 1.8848491890821606e-05, 1.3385357306106016e-05, 1.2544289347715676e-05, 1.8466671463102102e-05, 1.8661881767911837e-05, 1.4968009963922668e-05, 8.999315468827263e-06, 1.0690602721297182e-05, 8.055900252657011e-06, 3.16351288347505e-05, 2.247895099571906e-05, 4.017249011667445e-05, 7.213457865873352e-05, 6.327180017251521e-05, 9.369699546368793e-05, 0.00010289275087416172, 8.167078340193257e-05, 4.944878310197964e-05, 2.7665526431519538e-05, 3.0844257707940415e-05, 3.1183724786387756e-05, 1.5383408026536927e-05, 1.9008974049938843e-05, 1.5454665117431432e-05, 8.448021617368795e-06, 6.1480895965360105e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.1342676202730217e-05, "first_to_last_ratio": 0.9298538045037564, "imbalance_abs_log_ratio": 0.07272790465029173, "last_quartile_mean": 1.2198343597447092e-05, "mean": 1.9104475796893894e-05, "normalized": [ 0.48611542950870446, 0.6958890331865999, 0.7193778246429409, 0.443536817801586, 0.4695891169522356, 0.7667060760609765, 0.6873473082470735, 0.48118450474913316, 0.4543655526511291, 0.6811299166099339, 0.6646454976571455, 0.4541495149539672, 0.33836611538815065, 0.39892060594545364, 0.300624501076897, 1.0747424125435876, 0.797130210787533, 1.3941854251810624, 2.532754742399452, 1.9787761975331615, 2.748320565243497, 3.7309038097940537, 2.887865112295946, 1.7053168420610472, 0.9364859985915804, 1.057656867984351, 1.0602007618551181, 0.47533215874042833, 0.6202652229599234, 0.4947448929841209, 0.26849791503077114, 0.1948730485824397 ], "peak_layer": 22, "peak_normalized": 3.7309038097940537, "population_cv": 0.8569592439250643, "rest_mean": 1.4145171580484774e-05, "spike_contrast": 3.2438431944372783, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.588471856550314e-05, "top_five_layers": [ 22, 23, 21, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 9.286980457545724e-06, 1.3294595191837288e-05, 1.3743336239713244e-05, 8.473538400721736e-06, 8.97125391929876e-06, 1.4647517673438415e-05, 1.3131410014466383e-05, 9.19277772482019e-06, 8.680415703565814e-06, 1.3012630006414838e-05, 1.2697703823505435e-05, 8.676288416609168e-06, 6.464307261921931e-06, 7.621169061167166e-06, 5.74327350477688e-06, 2.0532390408334322e-05, 1.5228754818963353e-05, 2.663518171175383e-05, 4.8386951675638556e-05, 3.7803481973242015e-05, 5.250522372080013e-05, 7.127696153474972e-05, 5.517114914255217e-05, 3.25791843351908e-05, 1.789107409422286e-05, 2.0205980035825633e-05, 2.025457979470957e-05, 9.08097172214184e-06, 1.1849841939692851e-05, 9.451841833651997e-06, 5.129511919221841e-06, 3.722947440110147e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.0219679451402044e-05, "first_to_last_ratio": 0.8622296874614376, "imbalance_abs_log_ratio": 0.14823358504030823, "last_quartile_mean": 5.8243969306204235e-05, "mean": 8.878516183585816e-05, "normalized": [ 0.5866594709310274, 0.6211019320430468, 0.7097025860238361, 0.39191341369615046, 0.44573397193374353, 0.7357376579940132, 0.6020321723688816, 0.4321703611028182, 0.40170670285380555, 0.5630696276319809, 0.6035155111168472, 0.5482618086001224, 0.2706367956181458, 0.3291877010861888, 0.24164854743977685, 1.100466812721687, 0.7574718220240377, 1.3349314458317953, 2.386976535433516, 2.2364178778779267, 3.7197213872776747, 3.322432289382176, 2.7119382310195754, 1.6984839602598483, 0.9668042280017547, 1.0009347458556246, 1.0275821493704995, 0.5437250890941417, 0.6536934260198529, 0.556374259140604, 0.3002121346940273, 0.1987553455548734 ], "peak_layer": 21, "peak_normalized": 3.7197213872776747, "population_cv": 0.8943344260770331, "rest_mean": 6.438772248252967e-05, "spike_contrast": 3.4250525696675913, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000220531334343832, "top_five_layers": [ 21, 22, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 5.2086656069150195e-05, 5.514463555300608e-05, 6.301105895545334e-05, 3.479609586065635e-05, 3.957456283387728e-05, 6.532258703373373e-05, 5.345152385416441e-05, 3.8370315451174974e-05, 3.566559462342411e-05, 4.999222801416181e-05, 5.3583222324959934e-05, 4.8677513404982165e-05, 2.4028531697695144e-05, 2.9226983315311372e-05, 2.1454805391840637e-05, 9.770512406248599e-05, 6.725225830450654e-05, 0.00011852210445795208, 0.00021192809799686074, 0.00019856072321999818, 0.0003302560653537512, 0.00029498268850147724, 0.0002407798747299239, 0.0001508001732872799, 8.58378698467277e-05, 8.88681533979252e-05, 9.123404743149877e-05, 4.827472002943978e-05, 5.803827662020922e-05, 4.939777863910422e-05, 2.665438296389766e-05, 1.7646525520831347e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0596221424639225, 0.07634982466697693, 0.06968431174755096, 0.11288930475711823, 0.11885104328393936, 0.1495160460472107, 0.11554427444934845, 0.13194940984249115, 0.08884616196155548, 0.10897023230791092, 0.09843935817480087, 0.09431816637516022, 0.09025302529335022, 0.0895693451166153, 0.10807885229587555, 0.2204425185918808, 0.15380074083805084, 0.08548793941736221, 0.09071414917707443, 0.10860678553581238, 0.04808543995022774, 0.05631518363952637, 0.07238366454839706, 0.08572373539209366, 0.06546958535909653, 0.06323372572660446, 0.06869969516992569, 0.07729499042034149, 0.04753003641963005, 0.06484207510948181, 0.07835857570171356, 0.08203105628490448 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.6136328844804666e-06, "first_to_last_ratio": 0.1388409422044023, "imbalance_abs_log_ratio": 1.9744263017511179, "last_quartile_mean": 2.6027141757367644e-05, "mean": 1.5468678590480067e-05, "normalized": [ 0.478841741648447, 0.21344980083496803, 0.36542032760625354, 0.2153222147003958, 0.10396918122002749, 0.11898121205540443, 0.1673784156250151, 0.20551445783939368, 0.2659762543534721, 0.24072972609830148, 0.32074315460152014, 0.3998464374496061, 0.28459676525425437, 0.28657748114631404, 0.27675462844093746, 0.1575549602617813, 0.3238731831241911, 0.4768547933922055, 0.5952138273554176, 0.7441668021387481, 3.7970206059006064, 3.4384752815376007, 2.787460393051167, 2.2747146196513746, 2.288877373275197, 2.18568964820933, 2.055422183181528, 1.5564826251376447, 2.0236279536954616, 1.3877025596275139, 1.0837206429424298, 0.8790407486434917 ], "peak_layer": 21, "peak_normalized": 3.7970206059006064, "population_cv": 1.0347491651618592, "rest_mean": 9.97641066330996e-06, "spike_contrast": 4.523362852650103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.512692539719865e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 7.4070489972655196e-06, 3.3017863643181045e-06, 5.652569598169066e-06, 3.330750132590765e-06, 1.6082658476079814e-06, 1.8404821275908034e-06, 2.5891229142871452e-06, 3.1790370940143475e-06, 4.1143011912936345e-06, 3.723770760188927e-06, 4.961472768627573e-06, 6.185096026456449e-06, 4.402335889608366e-06, 4.432974947121693e-06, 4.281028395780595e-06, 2.4371670406253543e-06, 5.0098901738238055e-06, 7.376313533313805e-06, 9.207171387970448e-06, 1.1511277079989668e-05, 5.873489135410637e-05, 5.318866897141561e-05, 4.311832890380174e-05, 3.518682933645323e-05, 3.540590842021629e-05, 3.3809730666689575e-05, 3.1794665119377896e-05, 2.4076729459920898e-05, 3.1302850402425975e-05, 2.1465924874064513e-05, 1.6763726307544857e-05, 1.359759880870115e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.4505445051327115e-05, "first_to_last_ratio": 0.13916692625915444, "imbalance_abs_log_ratio": 1.9720811580258348, "last_quartile_mean": 0.00010423054845887236, "mean": 6.203866507803468e-05, "normalized": [ 0.4777447719884719, 0.21024222837857554, 0.3686322036929193, 0.2152923782079127, 0.10398351033326099, 0.1188088861677413, 0.16855731573325972, 0.20724253185602967, 0.2697828383859787, 0.2445416329818946, 0.32310538489568486, 0.39878685324314506, 0.28439552509962246, 0.28629442201468447, 0.27934695856398883, 0.16785764651758242, 0.3271642462653634, 0.4748504817782237, 0.5983260455063276, 0.739212269124694, 3.8074878097059237, 3.449181173181382, 2.775295719544032, 2.2631461806203244, 2.288355265686405, 2.179366722064293, 2.0601866927942867, 1.5606560046874582, 2.029916228241676, 1.373895253382417, 1.0798399654369917, 0.8685048539194496 ], "peak_layer": 21, "peak_normalized": 3.8074878097059237, "population_cv": 1.0347950762894988, "rest_mean": 4.0018463348840465e-05, "spike_contrast": 4.521606760318707, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018094775441568345, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.9638647902174853e-05, 1.3043147191638127e-05, 2.286944982188288e-05, 1.3356451745494269e-05, 6.450998171203537e-06, 7.37074469725485e-06, 1.0457070857228246e-05, 1.2857050023740157e-05, 1.6736967154429294e-05, 1.5171036466199439e-05, 2.0045026758452877e-05, 2.4740204025874846e-05, 1.7643518731347285e-05, 1.776132376107853e-05, 1.7330312402918935e-05, 1.041366431309143e-05, 2.0296833099564537e-05, 2.945909000118263e-05, 3.7119349144632e-05, 4.585974238580093e-05, 0.00023621146101504564, 0.0002139825955964625, 0.00017217564163729548, 0.00014040256792213768, 0.00014196650590747595, 0.00013520500215236098, 0.0001278112322324887, 9.682101517682895e-05, 0.00012593329302035272, 8.523462747689337e-05, 6.699182995362207e-05, 5.388088175095618e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.012863573734648526, "first_to_last_ratio": 0.13916692640099965, "imbalance_abs_log_ratio": 1.9720811570065897, "last_quartile_mean": 0.09243269264698029, "mean": 0.05501650820951909, "normalized": [ 0.47774476945422384, 0.21024223314992707, 0.3686322032999863, 0.2152923803280601, 0.10398351730346923, 0.11880888936358922, 0.168557316539217, 0.20724253070496107, 0.2697828471204318, 0.24454162479767508, 0.32310541341967935, 0.3987868618143491, 0.28439551814715514, 0.28629440598800215, 0.2793469622063591, 0.16785764609653073, 0.32716425359298057, 0.4748504780188051, 0.5983260263623125, 0.7392122842371308, 3.8074877818523305, 3.449181219200419, 2.775295592596114, 2.263146172837106, 2.2883552995642145, 2.1793667822100353, 2.060186762129678, 1.560655922165235, 2.029916146426396, 1.3738953075483749, 1.0798399969383916, 0.8685048545868602 ], "peak_layer": 21, "peak_normalized": 3.8074877818523305, "population_cv": 1.0347950719753642, "rest_mean": 0.03548877341327844, "spike_contrast": 4.521606713214234, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1604662761092186, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.02628384903073311, 0.01156679354608059, 0.020280856639146805, 0.011844635009765625, 0.005720810033380985, 0.006536450237035751, 0.009273434989154339, 0.011401760391891003, 0.014842510223388672, 0.013453826308250427, 0.017776131629943848, 0.02193986065685749, 0.015646448358893394, 0.01575091853737831, 0.015368694439530373, 0.009234941564500332, 0.0179994348436594, 0.026124615222215652, 0.0329178087413311, 0.040668878704309464, 0.20947468280792236, 0.1897619068622589, 0.15268707275390625, 0.12451039999723434, 0.12589731812477112, 0.1199011504650116, 0.11334428191184998, 0.08586183935403824, 0.11167889833450317, 0.07558692246675491, 0.05940902605652809, 0.04778210446238518 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.132429070746184e-07, "first_to_last_ratio": 0.13157058861916074, "imbalance_abs_log_ratio": 2.028211775804557, "last_quartile_mean": 6.941087036693716e-06, "mean": 4.019081015549375e-06, "normalized": [ 0.4719080582262415, 0.19563567332190476, 0.37589310034262197, 0.2187020951667458, 0.08866865629458083, 0.12479788224062963, 0.1539116204234896, 0.18829729202967996, 0.2114162989880101, 0.19477558566146955, 0.27471340176701525, 0.3560454871229287, 0.28155198404351195, 0.2833133731945887, 0.26477187349304765, 0.07414785991212178, 0.28510418059097403, 0.3651486236923762, 0.5232000418265924, 0.5799966034257934, 4.031742911648876, 3.4929475225280737, 2.7997174339527757, 2.3473254679510194, 2.3644425871309127, 2.264940124552736, 2.1454049683903906, 1.6160754654331098, 2.027589421813429, 1.4023816709528905, 1.0974092843705072, 0.8980234495109556 ], "peak_layer": 21, "peak_normalized": 4.031742911648876, "population_cv": 1.0854863513939026, "rest_mean": 2.5251475295607194e-06, "spike_contrast": 4.786382458212523, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2086321839888115e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.8966367179018562e-06, 7.862756206122867e-07, 1.5107448234630283e-06, 8.789814387455408e-07, 3.563665131878224e-07, 5.01572799294081e-07, 6.185832717164885e-07, 7.567820716758433e-07, 8.496992336404219e-07, 7.828188586245233e-07, 1.1040954177587992e-06, 1.4309756579677924e-06, 1.1315802339595393e-06, 1.1386593996576266e-06, 1.0641396102073486e-06, 2.980062561164232e-07, 1.1458567996669444e-06, 1.467561901336012e-06, 2.1027833554398967e-06, 2.331053337911726e-06, 1.620390139578376e-05, 1.4038439076102804e-05, 1.1252291187702212e-05, 9.434091225557495e-06, 9.502886314294301e-06, 9.102977855945937e-06, 8.622556379123125e-06, 6.4951382228173316e-06, 8.149046152539086e-06, 5.636285550281173e-06, 4.410576821101131e-06, 3.6092289974476444e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 9.655955466314481e-06, "first_to_last_ratio": 0.1540693296834623, "imbalance_abs_log_ratio": 1.8703525851251048, "last_quartile_mean": 6.267279468374909e-05, "mean": 3.9840255141143643e-05, "normalized": [ 0.5186483596750515, 0.22212115379322156, 0.38946276978980626, 0.19702341557402153, 0.11254194343112874, 0.1184124498685972, 0.17702354057452438, 0.2037008308619059, 0.29919017744364984, 0.2686201423560258, 0.3521019072470715, 0.44516471741513636, 0.3377857213600636, 0.3362681031102567, 0.3211205247678912, 0.1386776152858354, 0.34749360198352186, 0.5116110525159265, 0.632501071990601, 0.7925686032518788, 4.072805517742206, 3.6254666562082485, 2.7879110439949613, 2.206961143886179, 2.3448097066416045, 2.1048264561097376, 1.9085470681835104, 1.432339605184221, 1.8454197739428329, 1.2178876067716404, 0.9507433292729381, 0.7802443897658043 ], "peak_layer": 21, "peak_normalized": 4.072805517742206, "population_cv": 1.0495741603052116, "rest_mean": 2.5028601393623155e-05, "spike_contrast": 4.787450305085089, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011982318537775427, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.066308297798969e-05, 8.849363439367153e-06, 1.5516296116402373e-05, 7.849463145248592e-06, 4.4836997403763235e-06, 4.7175822146527935e-06, 7.0526630224776454e-06, 8.115493074001279e-06, 1.191981300507905e-05, 1.0701895007514395e-05, 1.4027829820406623e-05, 1.7735475921654142e-05, 1.3457469322020188e-05, 1.3397007023741025e-05, 1.2793523637810722e-05, 5.524951575353043e-06, 1.384423376293853e-05, 2.038271486526355e-05, 2.519900408515241e-05, 3.1576135370414704e-05, 0.00016226161096710712, 0.00014443951658904552, 0.0001110710873035714, 8.79258950590156e-05, 9.341781697003171e-05, 8.385682303924114e-05, 7.603700214531273e-05, 5.706477531930432e-05, 7.352199463639408e-05, 4.852095298701897e-05, 3.78778568119742e-05, 3.108513556071557e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 6.370375842834619e-06, "first_to_last_ratio": 0.19099125089091262, "imbalance_abs_log_ratio": 1.6555276588385337, "last_quartile_mean": 3.3354280958519666e-05, "mean": 2.3971603710037925e-05, "normalized": [ 0.5784128702594641, 0.24149517133975307, 0.42604453119872426, 0.19734517389086656, 0.13184453780025324, 0.12733703531507173, 0.2044711591588218, 0.2190235425972542, 0.3481067705913005, 0.31316189797316313, 0.3999261751321915, 0.5288777875214324, 0.4286351087415631, 0.4238951258244287, 0.39498954658816, 0.11192205557134918, 0.4033400615894943, 0.5955935820411911, 0.7169597117309988, 0.8670532520551248, 4.498418466436227, 3.907133635555405, 2.7489384705222526, 2.0558104028032402, 2.298695941814857, 1.891274127094267, 1.6215552432376346, 1.195951125397394, 1.5600224249476962, 1.0366900972214659, 0.8497028324036099, 0.6773721356453439 ], "peak_layer": 21, "peak_normalized": 4.498418466436227, "population_cv": 1.0695380403887063, "rest_mean": 1.4641325580871327e-05, "spike_contrast": 5.078440828109265, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.435510560753756e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.3865484106645454e-05, 5.789026545244269e-06, 1.0212970664724708e-05, 4.730680302600376e-06, 3.160525011480786e-06, 3.052472948184004e-06, 4.901501597487368e-06, 5.2503455663099885e-06, 8.34467755339574e-06, 7.506992915295996e-06, 9.586871783540118e-06, 1.267804873350542e-05, 1.0275070962961763e-05, 1.0161445970879868e-05, 9.468532880418934e-06, 2.682951162569225e-06, 9.668708116805647e-06, 1.4277333320933394e-05, 1.7186674085678533e-05, 2.078465695376508e-05, 0.00010783430479932576, 9.366025915369391e-05, 6.589646363863721e-05, 4.9281072278972715e-05, 5.5103428167058155e-05, 4.533687388175167e-05, 3.887127968482673e-05, 2.8668866434600204e-05, 3.7396239349618554e-05, 2.485112418071367e-05, 2.036873956967611e-05, 1.6237696399912238e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.835175428117509e-05, "first_to_last_ratio": 0.12494402924893692, "imbalance_abs_log_ratio": 2.079889407965475, "last_quartile_mean": 0.00022691563935950398, "mean": 0.00013135403207797935, "normalized": [ 0.45371617384329477, 0.2008095734378784, 0.34918619535186396, 0.18558037039021436, 0.09416845492909857, 0.10425965186662237, 0.1530284666175816, 0.18598938464103867, 0.2547076146729275, 0.23420270109644106, 0.309427949590735, 0.3818280972155079, 0.2607738377460453, 0.2628749102742167, 0.25605291926525836, 0.14748816253206562, 0.3004644499210074, 0.4368087743568899, 0.5630623449909483, 0.7166583299599412, 3.751756554644515, 3.44095092385939, 2.8305031368360156, 2.305606684861282, 2.416018818934567, 2.2538792330672948, 2.229722355101626, 1.6689559515627927, 2.0203916975982095, 1.3546173106278736, 1.030555164602258, 0.8459538056045994 ], "peak_layer": 21, "peak_normalized": 3.751756554644515, "population_cv": 1.0575612474642975, "rest_mean": 8.39457537006604e-05, "spike_contrast": 4.614393441469027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003873587353155017, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 5.95974488533102e-05, 2.637714715092443e-05, 4.5867014705436304e-05, 2.4376729925279506e-05, 1.2369406249490567e-05, 1.3694925655727275e-05, 2.0100906112929806e-05, 2.4430455596302636e-05, 3.345687218825333e-05, 3.076346911257133e-05, 4.064460881636478e-05, 5.015466012991965e-05, 3.425369504839182e-05, 3.452967939665541e-05, 3.3633583370829e-05, 1.937316483235918e-05, 3.9467216993216425e-05, 5.737659375881776e-05, 7.39605093258433e-05, 9.413596126250923e-05, 0.0004928083508275449, 0.000451982778031379, 0.00037179799983277917, 0.00030285073444247246, 0.0003173538134433329, 0.00029605612508021295, 0.00029288302175700665, 0.00021922409359831363, 0.00026538659585639834, 0.00017793444567359984, 0.0001353675761492923, 0.00011111944331787527 ] } } } } }, "learned": { "bits_per_byte": 1.877417995231997, "forward": { "activation_sha256": { "attention_branch_output": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", "mlp_branch_output": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", "post_attention_state": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", "post_mlp_state": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", "pre_attention_input": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", "pre_mlp_input": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a" }, "logits_sha256": "5f6747d51014b7e09e13dda43380fde8b7ba2972518bba8957bd86ff4ed7f868", "loss_nats": 1.3013269901275635, "mixer_summary_sha256": "9abaf96810a98e69acfbc813a9b903aa15d6279ddc462a3c26f1a1846711526f" }, "loss_nats": 1.3013269901275635, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.2200722098350525, "group": 1, "latest_source_mass": 0.06159520894289017, "layer": 1, "max_source_mass": 0.9384047985076904, "mixer_index": 1, "normalized_entropy": 0.3174970857665056, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9384047985076904, "median": 0.949082612991333, "p05": 0.8598991632461548, "p95": 0.9761404991149902 }, { "label": "current_group_partial", "mean": 0.06159520894289017, "median": 0.05091731622815132, "p05": 0.02385946735739708, "p95": 0.14010077714920044 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5651201605796814, "group": 1, "latest_source_mass": 0.3442652225494385, "layer": 2, "max_source_mass": 0.6557347774505615, "mixer_index": 2, "normalized_entropy": 0.815296053174681, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.6557347774505615, "median": 0.6636689305305481, "p05": 0.3472050428390503, "p95": 0.9190996289253235 }, { "label": "current_group_partial", "mean": 0.3442652225494385, "median": 0.3363310396671295, "p05": 0.08090042322874069, "p95": 0.6527949571609497 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4018177092075348, "group": 1, "latest_source_mass": 0.14357808232307434, "layer": 2, "max_source_mass": 0.856421947479248, "mixer_index": 3, "normalized_entropy": 0.5797004164150741, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.856421947479248, "median": 0.8660546541213989, "p05": 0.7540444135665894, "p95": 0.9155591726303101 }, { "label": "current_group_partial", "mean": 0.14357808232307434, "median": 0.13394534587860107, "p05": 0.08444082736968994, "p95": 0.24595558643341064 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.687358021736145, "group": 1, "latest_source_mass": 0.5312775373458862, "layer": 3, "max_source_mass": 0.5312775373458862, "mixer_index": 4, "normalized_entropy": 0.9916480092739848, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.46872249245643616, "median": 0.4674533009529114, "p05": 0.3992259204387665, "p95": 0.5431661605834961 }, { "label": "current_group_partial", "mean": 0.5312775373458862, "median": 0.5325466990470886, "p05": 0.45683377981185913, "p95": 0.6007740497589111 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.2678314447402954, "group": 1, "latest_source_mass": 0.07946884632110596, "layer": 3, "max_source_mass": 0.920531153678894, "mixer_index": 5, "normalized_entropy": 0.38639909712095066, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.920531153678894, "median": 0.9314625263214111, "p05": 0.8444089889526367, "p95": 0.9650533199310303 }, { "label": "current_group_partial", "mean": 0.07946884632110596, "median": 0.06853749603033066, "p05": 0.03494669869542122, "p95": 0.1555909812450409 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.657238245010376, "group": 1, "latest_source_mass": 0.3777551054954529, "layer": 4, "max_source_mass": 0.6222448348999023, "mixer_index": 6, "normalized_entropy": 0.948194356759035, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.6222448348999023, "median": 0.6208515167236328, "p05": 0.5314525961875916, "p95": 0.7065955400466919 }, { "label": "current_group_partial", "mean": 0.3777551054954529, "median": 0.3791484832763672, "p05": 0.2934044897556305, "p95": 0.4685474634170532 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5044373869895935, "group": 1, "latest_source_mass": 0.2207653522491455, "layer": 4, "max_source_mass": 0.7792346477508545, "mixer_index": 7, "normalized_entropy": 0.7277493166488734, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7792346477508545, "median": 0.7866228222846985, "p05": 0.6320486664772034, "p95": 0.9385759830474854 }, { "label": "current_group_partial", "mean": 0.2207653522491455, "median": 0.21337716281414032, "p05": 0.06142403930425644, "p95": 0.367951363325119 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.650418758392334, "group": 2, "latest_source_mass": 0.4602207541465759, "layer": 5, "max_source_mass": 0.5397792458534241, "mixer_index": 8, "normalized_entropy": 0.9383559172337771, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5397792458534241, "median": 0.5213949680328369, "p05": 0.34510529041290283, "p95": 0.7807042598724365 }, { "label": "completed_group_1", "mean": 0.4602207541465759, "median": 0.4786049723625183, "p05": 0.2192957103252411, "p95": 0.6548947095870972 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0032464265823364, "group": 2, "latest_source_mass": 0.37668097019195557, "layer": 5, "max_source_mass": 0.4337552785873413, "mixer_index": 9, "normalized_entropy": 0.913194251448444, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4337552785873413, "median": 0.42766237258911133, "p05": 0.2500072717666626, "p95": 0.6592159271240234 }, { "label": "completed_group_1", "mean": 0.1895637810230255, "median": 0.18978442251682281, "p05": 0.1284591555595398, "p95": 0.25696656107902527 }, { "label": "current_group_partial", "mean": 0.37668097019195557, "median": 0.3803442120552063, "p05": 0.1759825050830841, "p95": 0.568291425704956 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0385558605194092, "group": 2, "latest_source_mass": 0.4949595034122467, "layer": 6, "max_source_mass": 0.4949595034122467, "mixer_index": 10, "normalized_entropy": 0.9453342832879565, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.24478834867477417, "median": 0.23865866661071777, "p05": 0.19890648126602173, "p95": 0.32154381275177 }, { "label": "completed_group_1", "mean": 0.26025211811065674, "median": 0.26050686836242676, "p05": 0.23401768505573273, "p95": 0.2860541343688965 }, { "label": "current_group_partial", "mean": 0.4949595034122467, "median": 0.5017017722129822, "p05": 0.42678141593933105, "p95": 0.5358227491378784 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.031773328781128, "group": 2, "latest_source_mass": 0.33116668462753296, "layer": 6, "max_source_mass": 0.40370824933052063, "mixer_index": 11, "normalized_entropy": 0.9391605568439314, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.40370824933052063, "median": 0.38690465688705444, "p05": 0.18056227266788483, "p95": 0.7100913524627686 }, { "label": "completed_group_1", "mean": 0.2651250660419464, "median": 0.2730395793914795, "p05": 0.12547528743743896, "p95": 0.35344114899635315 }, { "label": "current_group_partial", "mean": 0.33116668462753296, "median": 0.33928292989730835, "p05": 0.1502782255411148, "p95": 0.47667479515075684 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0838522911071777, "group": 2, "latest_source_mass": 0.38127952814102173, "layer": 7, "max_source_mass": 0.38127952814102173, "mixer_index": 12, "normalized_entropy": 0.9865648712351232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35612422227859497, "median": 0.36401283740997314, "p05": 0.2946957051753998, "p95": 0.39828816056251526 }, { "label": "completed_group_1", "mean": 0.2625962495803833, "median": 0.2590169608592987, "p05": 0.24180665612220764, "p95": 0.2929879426956177 }, { "label": "current_group_partial", "mean": 0.38127952814102173, "median": 0.37846946716308594, "p05": 0.3489896059036255, "p95": 0.41648316383361816 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.96053147315979, "group": 2, "latest_source_mass": 0.3146248161792755, "layer": 7, "max_source_mass": 0.5271762013435364, "mixer_index": 13, "normalized_entropy": 0.874313425279704, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5271762013435364, "median": 0.5114221572875977, "p05": 0.3548204302787781, "p95": 0.7728348970413208 }, { "label": "completed_group_1", "mean": 0.1581989824771881, "median": 0.1618250608444214, "p05": 0.07541970163583755, "p95": 0.21691158413887024 }, { "label": "current_group_partial", "mean": 0.3146248161792755, "median": 0.32365772128105164, "p05": 0.14578065276145935, "p95": 0.4483411908149719 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0704245567321777, "group": 2, "latest_source_mass": 0.38260507583618164, "layer": 8, "max_source_mass": 0.38260507583618164, "mixer_index": 14, "normalized_entropy": 0.9743424206822726, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3788670599460602, "median": 0.39538437128067017, "p05": 0.27489906549453735, "p95": 0.43731769919395447 }, { "label": "completed_group_1", "mean": 0.23852786421775818, "median": 0.2294245958328247, "p05": 0.20573674142360687, "p95": 0.29735344648361206 }, { "label": "current_group_partial", "mean": 0.38260507583618164, "median": 0.37669745087623596, "p05": 0.34335845708847046, "p95": 0.435106486082077 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9674104452133179, "group": 2, "latest_source_mass": 0.3281455934047699, "layer": 8, "max_source_mass": 0.48829177021980286, "mixer_index": 15, "normalized_entropy": 0.8805749354816949, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.48829177021980286, "median": 0.44956350326538086, "p05": 0.2823859453201294, "p95": 0.8209846615791321 }, { "label": "completed_group_1", "mean": 0.18356263637542725, "median": 0.18210050463676453, "p05": 0.05149933695793152, "p95": 0.2906108498573303 }, { "label": "current_group_partial", "mean": 0.3281455934047699, "median": 0.34843379259109497, "p05": 0.12376503646373749, "p95": 0.46937939524650574 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0832805633544922, "group": 3, "latest_source_mass": 0.31498202681541443, "layer": 9, "max_source_mass": 0.40388280153274536, "mixer_index": 16, "normalized_entropy": 0.9860444622076776, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.40388280153274536, "median": 0.403384804725647, "p05": 0.3489384353160858, "p95": 0.4601682126522064 }, { "label": "completed_group_1", "mean": 0.2811351716518402, "median": 0.28186845779418945, "p05": 0.25417307019233704, "p95": 0.30605652928352356 }, { "label": "completed_group_2", "mean": 0.31498202681541443, "median": 0.3153840899467468, "p05": 0.2613564431667328, "p95": 0.3658811151981354 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2917464971542358, "group": 3, "latest_source_mass": 0.33769795298576355, "layer": 9, "max_source_mass": 0.33769795298576355, "mixer_index": 17, "normalized_entropy": 0.9317981327650527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3349480926990509, "median": 0.3352771997451782, "p05": 0.18474003672599792, "p95": 0.48752525448799133 }, { "label": "completed_group_1", "mean": 0.1244347020983696, "median": 0.11896269023418427, "p05": 0.07986846566200256, "p95": 0.1843445897102356 }, { "label": "completed_group_2", "mean": 0.20291924476623535, "median": 0.20060715079307556, "p05": 0.1253921389579773, "p95": 0.2836313247680664 }, { "label": "current_group_partial", "mean": 0.33769795298576355, "median": 0.33908897638320923, "p05": 0.26979348063468933, "p95": 0.4016563892364502 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3533791303634644, "group": 3, "latest_source_mass": 0.28473395109176636, "layer": 10, "max_source_mass": 0.32115638256073, "mixer_index": 18, "normalized_entropy": 0.976256679908994, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.32115638256073, "median": 0.3355404734611511, "p05": 0.22348493337631226, "p95": 0.3863494396209717 }, { "label": "completed_group_1", "mean": 0.20602071285247803, "median": 0.2052205502986908, "p05": 0.1802767962217331, "p95": 0.23218519985675812 }, { "label": "completed_group_2", "mean": 0.18808895349502563, "median": 0.18706050515174866, "p05": 0.16633892059326172, "p95": 0.2130376398563385 }, { "label": "current_group_partial", "mean": 0.28473395109176636, "median": 0.275795578956604, "p05": 0.22748424112796783, "p95": 0.3660258948802948 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.308367371559143, "group": 3, "latest_source_mass": 0.29719752073287964, "layer": 10, "max_source_mass": 0.3113970160484314, "mixer_index": 19, "normalized_entropy": 0.9437875593046517, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3113970160484314, "median": 0.33913683891296387, "p05": 0.0915825292468071, "p95": 0.47554588317871094 }, { "label": "completed_group_1", "mean": 0.16179001331329346, "median": 0.16501447558403015, "p05": 0.10682816803455353, "p95": 0.20702867209911346 }, { "label": "completed_group_2", "mean": 0.2296154499053955, "median": 0.2261994481086731, "p05": 0.14510810375213623, "p95": 0.32595163583755493 }, { "label": "current_group_partial", "mean": 0.29719752073287964, "median": 0.2761377692222595, "p05": 0.194716677069664, "p95": 0.4515363574028015 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3437637090682983, "group": 3, "latest_source_mass": 0.3098427951335907, "layer": 11, "max_source_mass": 0.3217787742614746, "mixer_index": 20, "normalized_entropy": 0.969320619599697, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3217787742614746, "median": 0.31754863262176514, "p05": 0.25967735052108765, "p95": 0.40468457341194153 }, { "label": "completed_group_1", "mean": 0.1920534074306488, "median": 0.1915990114212036, "p05": 0.16391108930110931, "p95": 0.22057634592056274 }, { "label": "completed_group_2", "mean": 0.1763250231742859, "median": 0.1779440939426422, "p05": 0.14714840054512024, "p95": 0.1996835470199585 }, { "label": "current_group_partial", "mean": 0.3098427951335907, "median": 0.31089526414871216, "p05": 0.2459525167942047, "p95": 0.36900240182876587 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2280659675598145, "group": 3, "latest_source_mass": 0.29383569955825806, "layer": 11, "max_source_mass": 0.42534634470939636, "mixer_index": 21, "normalized_entropy": 0.8858623406415255, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.42534634470939636, "median": 0.4574331045150757, "p05": 0.17354995012283325, "p95": 0.6023476123809814 }, { "label": "completed_group_1", "mean": 0.14848899841308594, "median": 0.14150214195251465, "p05": 0.07509882748126984, "p95": 0.23531809449195862 }, { "label": "completed_group_2", "mean": 0.13232898712158203, "median": 0.13064970076084137, "p05": 0.07000484317541122, "p95": 0.19124652445316315 }, { "label": "current_group_partial", "mean": 0.29383569955825806, "median": 0.27858495712280273, "p05": 0.187642902135849, "p95": 0.4577033817768097 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3462648391723633, "group": 3, "latest_source_mass": 0.3101534843444824, "layer": 12, "max_source_mass": 0.3221249580383301, "mixer_index": 22, "normalized_entropy": 0.9711248035985732, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3221249580383301, "median": 0.3120178282260895, "p05": 0.2763444483280182, "p95": 0.3882659375667572 }, { "label": "completed_group_1", "mean": 0.18557845056056976, "median": 0.1851869523525238, "p05": 0.16533124446868896, "p95": 0.20681798458099365 }, { "label": "completed_group_2", "mean": 0.18214309215545654, "median": 0.18278372287750244, "p05": 0.15564095973968506, "p95": 0.2072875201702118 }, { "label": "current_group_partial", "mean": 0.3101534843444824, "median": 0.31112557649612427, "p05": 0.2608441412448883, "p95": 0.36039745807647705 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.147221326828003, "group": 3, "latest_source_mass": 0.2585829496383667, "layer": 12, "max_source_mass": 0.5053804516792297, "mixer_index": 23, "normalized_entropy": 0.8275452595084083, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5053804516792297, "median": 0.48097139596939087, "p05": 0.30125147104263306, "p95": 0.7477225065231323 }, { "label": "completed_group_1", "mean": 0.12089012563228607, "median": 0.11902710795402527, "p05": 0.05662901699542999, "p95": 0.1844676434993744 }, { "label": "completed_group_2", "mean": 0.11514648050069809, "median": 0.11994394659996033, "p05": 0.05982710421085358, "p95": 0.16130289435386658 }, { "label": "current_group_partial", "mean": 0.2585829496383667, "median": 0.25007814168930054, "p05": 0.12433226406574249, "p95": 0.42273372411727905 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3729398250579834, "group": 4, "latest_source_mass": 0.22271886467933655, "layer": 13, "max_source_mass": 0.30785465240478516, "mixer_index": 24, "normalized_entropy": 0.9903667385250569, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.30785465240478516, "median": 0.3129041790962219, "p05": 0.25217515230178833, "p95": 0.3537779450416565 }, { "label": "completed_group_1", "mean": 0.23079943656921387, "median": 0.22906282544136047, "p05": 0.19741898775100708, "p95": 0.26931890845298767 }, { "label": "completed_group_2", "mean": 0.23862707614898682, "median": 0.23815912008285522, "p05": 0.21215641498565674, "p95": 0.2663886845111847 }, { "label": "completed_group_3", "mean": 0.22271886467933655, "median": 0.22075486183166504, "p05": 0.186451256275177, "p95": 0.2638319730758667 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5232064723968506, "group": 4, "latest_source_mass": 0.24461600184440613, "layer": 13, "max_source_mass": 0.3136816620826721, "mixer_index": 25, "normalized_entropy": 0.9464213938474744, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3136816620826721, "median": 0.326117604970932, "p05": 0.20238417387008667, "p95": 0.4362232983112335 }, { "label": "completed_group_1", "mean": 0.11285123229026794, "median": 0.11373943835496902, "p05": 0.08906605839729309, "p95": 0.13395145535469055 }, { "label": "completed_group_2", "mean": 0.13196846842765808, "median": 0.1334267258644104, "p05": 0.10183953493833542, "p95": 0.16085752844810486 }, { "label": "completed_group_3", "mean": 0.19688265025615692, "median": 0.1940557211637497, "p05": 0.13830327987670898, "p95": 0.2671412229537964 }, { "label": "current_group_partial", "mean": 0.24461600184440613, "median": 0.2419893592596054, "p05": 0.18994617462158203, "p95": 0.3077915906906128 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4876129627227783, "group": 4, "latest_source_mass": 0.396677166223526, "layer": 14, "max_source_mass": 0.396677166223526, "mixer_index": 26, "normalized_entropy": 0.9243059028433878, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2035508155822754, "median": 0.20008960366249084, "p05": 0.16096048057079315, "p95": 0.255715548992157 }, { "label": "completed_group_1", "mean": 0.15372203290462494, "median": 0.15584711730480194, "p05": 0.12048184871673584, "p95": 0.18010932207107544 }, { "label": "completed_group_2", "mean": 0.13167491555213928, "median": 0.13113251328468323, "p05": 0.11609883606433868, "p95": 0.14900363981723785 }, { "label": "completed_group_3", "mean": 0.11437506973743439, "median": 0.11201842874288559, "p05": 0.09515875577926636, "p95": 0.1399364471435547 }, { "label": "current_group_partial", "mean": 0.396677166223526, "median": 0.39533013105392456, "p05": 0.3409242630004883, "p95": 0.4550648033618927 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4587602615356445, "group": 4, "latest_source_mass": 0.33461427688598633, "layer": 14, "max_source_mass": 0.33461427688598633, "mixer_index": 27, "normalized_entropy": 0.906378711639412, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28519001603126526, "median": 0.28675830364227295, "p05": 0.1500462144613266, "p95": 0.45491817593574524 }, { "label": "completed_group_1", "mean": 0.10532153397798538, "median": 0.09640030562877655, "p05": 0.06345808506011963, "p95": 0.1692449301481247 }, { "label": "completed_group_2", "mean": 0.09675674885511398, "median": 0.09718051552772522, "p05": 0.058796193450689316, "p95": 0.13363884389400482 }, { "label": "completed_group_3", "mean": 0.17811740934848785, "median": 0.1759425848722458, "p05": 0.1284305453300476, "p95": 0.23726706206798553 }, { "label": "current_group_partial", "mean": 0.33461427688598633, "median": 0.33546411991119385, "p05": 0.23162710666656494, "p95": 0.4484652876853943 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4565696716308594, "group": 4, "latest_source_mass": 0.4221859574317932, "layer": 15, "max_source_mass": 0.4221859574317932, "mixer_index": 28, "normalized_entropy": 0.9050176216042753, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20232918858528137, "median": 0.1998670995235443, "p05": 0.15170496702194214, "p95": 0.26285088062286377 }, { "label": "completed_group_1", "mean": 0.14928826689720154, "median": 0.14749380946159363, "p05": 0.12863489985466003, "p95": 0.17874351143836975 }, { "label": "completed_group_2", "mean": 0.12029382586479187, "median": 0.11950792372226715, "p05": 0.10211217403411865, "p95": 0.14088989794254303 }, { "label": "completed_group_3", "mean": 0.1059027910232544, "median": 0.10411787033081055, "p05": 0.08704904466867447, "p95": 0.13138693571090698 }, { "label": "current_group_partial", "mean": 0.4221859574317932, "median": 0.4195709228515625, "p05": 0.35523390769958496, "p95": 0.4969034492969513 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4788098335266113, "group": 4, "latest_source_mass": 0.34753501415252686, "layer": 15, "max_source_mass": 0.34753501415252686, "mixer_index": 29, "normalized_entropy": 0.9188362111403675, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2697942852973938, "median": 0.25651901960372925, "p05": 0.146310955286026, "p95": 0.42084938287734985 }, { "label": "completed_group_1", "mean": 0.11948063224554062, "median": 0.11549928039312363, "p05": 0.0849025622010231, "p95": 0.16021209955215454 }, { "label": "completed_group_2", "mean": 0.11090938746929169, "median": 0.11184529960155487, "p05": 0.08139486610889435, "p95": 0.13886672258377075 }, { "label": "completed_group_3", "mean": 0.15228068828582764, "median": 0.15164361894130707, "p05": 0.11261995136737823, "p95": 0.19522227346897125 }, { "label": "current_group_partial", "mean": 0.34753501415252686, "median": 0.3453425168991089, "p05": 0.25332537293434143, "p95": 0.44237542152404785 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5018762350082397, "group": 4, "latest_source_mass": 0.31088152527809143, "layer": 16, "max_source_mass": 0.31088152527809143, "mixer_index": 30, "normalized_entropy": 0.9331681721954809, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.208971306681633, "median": 0.20881521701812744, "p05": 0.16508029401302338, "p95": 0.2516583502292633 }, { "label": "completed_group_1", "mean": 0.27331358194351196, "median": 0.26948875188827515, "p05": 0.23055149614810944, "p95": 0.33120542764663696 }, { "label": "completed_group_2", "mean": 0.12766750156879425, "median": 0.1261082887649536, "p05": 0.10924770683050156, "p95": 0.1494429111480713 }, { "label": "completed_group_3", "mean": 0.07916608452796936, "median": 0.07811431586742401, "p05": 0.0561223030090332, "p95": 0.10760308802127838 }, { "label": "current_group_partial", "mean": 0.31088152527809143, "median": 0.3123162090778351, "p05": 0.2707694172859192, "p95": 0.3467172384262085 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5627843141555786, "group": 4, "latest_source_mass": 0.22652018070220947, "layer": 16, "max_source_mass": 0.27452588081359863, "mixer_index": 31, "normalized_entropy": 0.9710124895666443, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27452588081359863, "median": 0.2678954601287842, "p05": 0.19414913654327393, "p95": 0.3916982412338257 }, { "label": "completed_group_1", "mean": 0.21448183059692383, "median": 0.21617943048477173, "p05": 0.17793960869312286, "p95": 0.24279263615608215 }, { "label": "completed_group_2", "mean": 0.143874391913414, "median": 0.1446641981601715, "p05": 0.11340779811143875, "p95": 0.17275942862033844 }, { "label": "completed_group_3", "mean": 0.14059773087501526, "median": 0.13659776747226715, "p05": 0.11039599776268005, "p95": 0.18301524221897125 }, { "label": "current_group_partial", "mean": 0.22652018070220947, "median": 0.22071748971939087, "p05": 0.16668984293937683, "p95": 0.293569415807724 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.580754280090332, "group": 5, "latest_source_mass": 0.24107450246810913, "layer": 17, "max_source_mass": 0.2518634498119354, "mixer_index": 32, "normalized_entropy": 0.9821778571747528, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2518634498119354, "median": 0.24593469500541687, "p05": 0.21242231130599976, "p95": 0.3016672730445862 }, { "label": "completed_group_1", "mean": 0.18758200109004974, "median": 0.1801876276731491, "p05": 0.15723729133605957, "p95": 0.24892441928386688 }, { "label": "completed_group_2", "mean": 0.13903990387916565, "median": 0.13839681446552277, "p05": 0.12513971328735352, "p95": 0.15491701662540436 }, { "label": "completed_group_3", "mean": 0.18044015765190125, "median": 0.1819951832294464, "p05": 0.14810611307621002, "p95": 0.20801132917404175 }, { "label": "completed_group_4", "mean": 0.24107450246810913, "median": 0.24384509027004242, "p05": 0.1871698796749115, "p95": 0.2886185050010681 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7564777135849, "group": 5, "latest_source_mass": 0.2205006182193756, "layer": 17, "max_source_mass": 0.2205006182193756, "mixer_index": 33, "normalized_entropy": 0.9803088772521708, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15440437197685242, "median": 0.15525281429290771, "p05": 0.11778812110424042, "p95": 0.18865147233009338 }, { "label": "completed_group_1", "mean": 0.19411899149417877, "median": 0.2002902328968048, "p05": 0.14482440054416656, "p95": 0.2256385236978531 }, { "label": "completed_group_2", "mean": 0.17806550860404968, "median": 0.1781877875328064, "p05": 0.15101563930511475, "p95": 0.20418882369995117 }, { "label": "completed_group_3", "mean": 0.14696575701236725, "median": 0.14620032906532288, "p05": 0.10836094617843628, "p95": 0.1874053031206131 }, { "label": "completed_group_4", "mean": 0.10594475269317627, "median": 0.10578415542840958, "p05": 0.08951808512210846, "p95": 0.1216079369187355 }, { "label": "current_group_partial", "mean": 0.2205006182193756, "median": 0.21638520061969757, "p05": 0.16007332503795624, "p95": 0.2944811284542084 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.5899518728256226, "group": 5, "latest_source_mass": 0.4157395362854004, "layer": 18, "max_source_mass": 0.4157395362854004, "mixer_index": 34, "normalized_entropy": 0.8873690359290373, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1636897772550583, "median": 0.16218292713165283, "p05": 0.1178273856639862, "p95": 0.21766462922096252 }, { "label": "completed_group_1", "mean": 0.11789833009243011, "median": 0.11790023744106293, "p05": 0.1032494530081749, "p95": 0.13267967104911804 }, { "label": "completed_group_2", "mean": 0.07030131667852402, "median": 0.0698043555021286, "p05": 0.06258689612150192, "p95": 0.07938572019338608 }, { "label": "completed_group_3", "mean": 0.09238425642251968, "median": 0.09013497084379196, "p05": 0.0768275037407875, "p95": 0.1164383590221405 }, { "label": "completed_group_4", "mean": 0.1399867981672287, "median": 0.1400289535522461, "p05": 0.12679682672023773, "p95": 0.15317322313785553 }, { "label": "current_group_partial", "mean": 0.4157395362854004, "median": 0.4171099066734314, "p05": 0.367803692817688, "p95": 0.4605288803577423 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.2894535064697266, "group": 5, "latest_source_mass": 0.5977407693862915, "layer": 18, "max_source_mass": 0.5977407693862915, "mixer_index": 35, "normalized_entropy": 0.7196577044045218, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1212274506688118, "median": 0.10892187058925629, "p05": 0.07249757647514343, "p95": 0.2145335078239441 }, { "label": "completed_group_1", "mean": 0.06944876164197922, "median": 0.0678764283657074, "p05": 0.05350388586521149, "p95": 0.08999023586511612 }, { "label": "completed_group_2", "mean": 0.05726535990834236, "median": 0.0559837631881237, "p05": 0.0409373864531517, "p95": 0.07825973629951477 }, { "label": "completed_group_3", "mean": 0.09096579253673553, "median": 0.08796361088752747, "p05": 0.06407731771469116, "p95": 0.12661314010620117 }, { "label": "completed_group_4", "mean": 0.06335186958312988, "median": 0.06283852458000183, "p05": 0.05032140761613846, "p95": 0.0781375840306282 }, { "label": "current_group_partial", "mean": 0.5977407693862915, "median": 0.6085352301597595, "p05": 0.4878399074077606, "p95": 0.6737192869186401 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6581614017486572, "group": 5, "latest_source_mass": 0.35773903131484985, "layer": 19, "max_source_mass": 0.35773903131484985, "mixer_index": 36, "normalized_entropy": 0.9254374988530375, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18132120370864868, "median": 0.1806161105632782, "p05": 0.1410265862941742, "p95": 0.22331592440605164 }, { "label": "completed_group_1", "mean": 0.12711825966835022, "median": 0.12722507119178772, "p05": 0.11291128396987915, "p95": 0.14178316295146942 }, { "label": "completed_group_2", "mean": 0.08560527861118317, "median": 0.08513904362916946, "p05": 0.07705061137676239, "p95": 0.09564155340194702 }, { "label": "completed_group_3", "mean": 0.09748885035514832, "median": 0.0953599065542221, "p05": 0.08346174657344818, "p95": 0.1194010004401207 }, { "label": "completed_group_4", "mean": 0.15072739124298096, "median": 0.150638610124588, "p05": 0.13762864470481873, "p95": 0.1634974181652069 }, { "label": "current_group_partial", "mean": 0.35773903131484985, "median": 0.35794761776924133, "p05": 0.32225728034973145, "p95": 0.39409786462783813 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3699336051940918, "group": 5, "latest_source_mass": 0.5385624766349792, "layer": 19, "max_source_mass": 0.5385624766349792, "mixer_index": 37, "normalized_entropy": 0.7645745027284836, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13604649901390076, "median": 0.11998516321182251, "p05": 0.06740576773881912, "p95": 0.26897257566452026 }, { "label": "completed_group_1", "mean": 0.10151474922895432, "median": 0.0991508960723877, "p05": 0.07438431680202484, "p95": 0.13600951433181763 }, { "label": "completed_group_2", "mean": 0.07715044170618057, "median": 0.0767412930727005, "p05": 0.054624155163764954, "p95": 0.09943743050098419 }, { "label": "completed_group_3", "mean": 0.11300846189260483, "median": 0.11003986746072769, "p05": 0.08064281940460205, "p95": 0.1516837179660797 }, { "label": "completed_group_4", "mean": 0.033717282116413116, "median": 0.03286907076835632, "p05": 0.024589763954281807, "p95": 0.04592139273881912 }, { "label": "current_group_partial", "mean": 0.5385624766349792, "median": 0.5415897369384766, "p05": 0.4144754409790039, "p95": 0.6506978273391724 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6926933526992798, "group": 5, "latest_source_mass": 0.3198680281639099, "layer": 20, "max_source_mass": 0.3198680281639099, "mixer_index": 38, "normalized_entropy": 0.9447101476341264, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19425281882286072, "median": 0.19403350353240967, "p05": 0.139950692653656, "p95": 0.2494640350341797 }, { "label": "completed_group_1", "mean": 0.14001017808914185, "median": 0.14058321714401245, "p05": 0.12015820294618607, "p95": 0.158815398812294 }, { "label": "completed_group_2", "mean": 0.09434760361909866, "median": 0.09424005448818207, "p05": 0.08333250135183334, "p95": 0.10548526048660278 }, { "label": "completed_group_3", "mean": 0.10331346094608307, "median": 0.10153411328792572, "p05": 0.08690286427736282, "p95": 0.12658309936523438 }, { "label": "completed_group_4", "mean": 0.14820793271064758, "median": 0.14894580841064453, "p05": 0.13278263807296753, "p95": 0.16199614107608795 }, { "label": "current_group_partial", "mean": 0.3198680281639099, "median": 0.3201481103897095, "p05": 0.2800636887550354, "p95": 0.3604968786239624 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4148362874984741, "group": 5, "latest_source_mass": 0.4996800422668457, "layer": 20, "max_source_mass": 0.4996800422668457, "mixer_index": 39, "normalized_entropy": 0.789635166883214, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16652248799800873, "median": 0.14292314648628235, "p05": 0.08186839520931244, "p95": 0.350705623626709 }, { "label": "completed_group_1", "mean": 0.10329679399728775, "median": 0.10057851672172546, "p05": 0.07478320598602295, "p95": 0.13858266174793243 }, { "label": "completed_group_2", "mean": 0.08063912391662598, "median": 0.0807541012763977, "p05": 0.056308042258024216, "p95": 0.10469165444374084 }, { "label": "completed_group_3", "mean": 0.1165856122970581, "median": 0.11373114585876465, "p05": 0.08339665830135345, "p95": 0.15619920194149017 }, { "label": "completed_group_4", "mean": 0.03327595070004463, "median": 0.032142095267772675, "p05": 0.022759035229682922, "p95": 0.04896518588066101 }, { "label": "current_group_partial", "mean": 0.4996800422668457, "median": 0.5001581907272339, "p05": 0.3463003933429718, "p95": 0.6266770362854004 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.740808129310608, "group": 6, "latest_source_mass": 0.24231041967868805, "layer": 21, "max_source_mass": 0.24231041967868805, "mixer_index": 40, "normalized_entropy": 0.9715635157550481, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21459883451461792, "median": 0.21599377691745758, "p05": 0.1570315659046173, "p95": 0.27208006381988525 }, { "label": "completed_group_1", "mean": 0.15640269219875336, "median": 0.1569681316614151, "p05": 0.13785089552402496, "p95": 0.1749057024717331 }, { "label": "completed_group_2", "mean": 0.10570268332958221, "median": 0.10564640164375305, "p05": 0.09223169088363647, "p95": 0.11907710134983063 }, { "label": "completed_group_3", "mean": 0.11097013205289841, "median": 0.10862545669078827, "p05": 0.09454305469989777, "p95": 0.13635265827178955 }, { "label": "completed_group_4", "mean": 0.17001524567604065, "median": 0.17010903358459473, "p05": 0.15088173747062683, "p95": 0.18767674267292023 }, { "label": "completed_group_5", "mean": 0.24231041967868805, "median": 0.24274763464927673, "p05": 0.2087050974369049, "p95": 0.27555277943611145 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3179612159729004, "group": 6, "latest_source_mass": 0.603651762008667, "layer": 21, "max_source_mass": 0.603651762008667, "mixer_index": 41, "normalized_entropy": 0.6772980841960945, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08068332821130753, "median": 0.06917878985404968, "p05": 0.039396293461322784, "p95": 0.17595769464969635 }, { "label": "completed_group_1", "mean": 0.049232251942157745, "median": 0.04861311614513397, "p05": 0.03742668777704239, "p95": 0.06363335251808167 }, { "label": "completed_group_2", "mean": 0.045073024928569794, "median": 0.04482679069042206, "p05": 0.03333880379796028, "p95": 0.057495348155498505 }, { "label": "completed_group_3", "mean": 0.06111945956945419, "median": 0.059528060257434845, "p05": 0.042710162699222565, "p95": 0.08362975716590881 }, { "label": "completed_group_4", "mean": 0.028317855671048164, "median": 0.027924545109272003, "p05": 0.022549759596586227, "p95": 0.03542562201619148 }, { "label": "completed_group_5", "mean": 0.13192236423492432, "median": 0.12939827144145966, "p05": 0.10056710243225098, "p95": 0.17195004224777222 }, { "label": "current_group_partial", "mean": 0.603651762008667, "median": 0.6094026565551758, "p05": 0.5065152645111084, "p95": 0.6787958145141602 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.5319744348526, "group": 6, "latest_source_mass": 0.5132461190223694, "layer": 22, "max_source_mass": 0.5132461190223694, "mixer_index": 42, "normalized_entropy": 0.7872791226235868, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12406684458255768, "median": 0.1289612352848053, "p05": 0.06980825215578079, "p95": 0.16972985863685608 }, { "label": "completed_group_1", "mean": 0.07462044060230255, "median": 0.07392798364162445, "p05": 0.06016036123037338, "p95": 0.0917147696018219 }, { "label": "completed_group_2", "mean": 0.05378449708223343, "median": 0.053309448063373566, "p05": 0.04370884597301483, "p95": 0.06546851992607117 }, { "label": "completed_group_3", "mean": 0.05575074627995491, "median": 0.053552836179733276, "p05": 0.0437602773308754, "p95": 0.07444392144680023 }, { "label": "completed_group_4", "mean": 0.07357081025838852, "median": 0.07329098880290985, "p05": 0.06283499300479889, "p95": 0.08542977273464203 }, { "label": "completed_group_5", "mean": 0.10496053099632263, "median": 0.1038457602262497, "p05": 0.08522139489650726, "p95": 0.1291121542453766 }, { "label": "current_group_partial", "mean": 0.5132461190223694, "median": 0.5090781450271606, "p05": 0.4398295283317566, "p95": 0.59932541847229 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.2461028099060059, "group": 6, "latest_source_mass": 0.6375080347061157, "layer": 22, "max_source_mass": 0.6375080347061157, "mixer_index": 43, "normalized_entropy": 0.640370168432985, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08556011319160461, "median": 0.07947102189064026, "p05": 0.043556369841098785, "p95": 0.16172736883163452 }, { "label": "completed_group_1", "mean": 0.0446893647313118, "median": 0.04398339241743088, "p05": 0.03128381073474884, "p95": 0.05990619957447052 }, { "label": "completed_group_2", "mean": 0.04084867238998413, "median": 0.04042873531579971, "p05": 0.02942952886223793, "p95": 0.05392487719655037 }, { "label": "completed_group_3", "mean": 0.05386577919125557, "median": 0.052273523062467575, "p05": 0.033789411187171936, "p95": 0.076903335750103 }, { "label": "completed_group_4", "mean": 0.025119207799434662, "median": 0.024625632911920547, "p05": 0.019138969480991364, "p95": 0.03265690058469772 }, { "label": "completed_group_5", "mean": 0.11240880191326141, "median": 0.11003861576318741, "p05": 0.07852104306221008, "p95": 0.155371755361557 }, { "label": "current_group_partial", "mean": 0.6375080347061157, "median": 0.6392797231674194, "p05": 0.5317062735557556, "p95": 0.7386638522148132 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.6323832273483276, "group": 6, "latest_source_mass": 0.4532663822174072, "layer": 23, "max_source_mass": 0.4532663822174072, "mixer_index": 44, "normalized_entropy": 0.8388790346464895, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1418324112892151, "median": 0.14852997660636902, "p05": 0.07787702977657318, "p95": 0.1923113763332367 }, { "label": "completed_group_1", "mean": 0.08991365134716034, "median": 0.08918912708759308, "p05": 0.0752246081829071, "p95": 0.10654127597808838 }, { "label": "completed_group_2", "mean": 0.06188817322254181, "median": 0.06161077320575714, "p05": 0.04994015768170357, "p95": 0.07481888681650162 }, { "label": "completed_group_3", "mean": 0.05739111453294754, "median": 0.0548475980758667, "p05": 0.04428064078092575, "p95": 0.07816634327173233 }, { "label": "completed_group_4", "mean": 0.09848874807357788, "median": 0.09768162667751312, "p05": 0.08320866525173187, "p95": 0.11606857925653458 }, { "label": "completed_group_5", "mean": 0.0972195416688919, "median": 0.09613433480262756, "p05": 0.07767169177532196, "p95": 0.12004032731056213 }, { "label": "current_group_partial", "mean": 0.4532663822174072, "median": 0.45108917355537415, "p05": 0.37653055787086487, "p95": 0.5389904975891113 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.301480770111084, "group": 6, "latest_source_mass": 0.6134276390075684, "layer": 23, "max_source_mass": 0.6134276390075684, "mixer_index": 45, "normalized_entropy": 0.6688288103861927, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09513165056705475, "median": 0.08380547165870667, "p05": 0.055099427700042725, "p95": 0.1882638931274414 }, { "label": "completed_group_1", "mean": 0.050291214138269424, "median": 0.04947858303785324, "p05": 0.03757611662149429, "p95": 0.06516233086585999 }, { "label": "completed_group_2", "mean": 0.045042067766189575, "median": 0.0443364642560482, "p05": 0.031873252242803574, "p95": 0.06115761026740074 }, { "label": "completed_group_3", "mean": 0.05902485549449921, "median": 0.05624900758266449, "p05": 0.038259170949459076, "p95": 0.08772744238376617 }, { "label": "completed_group_4", "mean": 0.02615261636674404, "median": 0.02565642073750496, "p05": 0.01966264471411705, "p95": 0.03415588289499283 }, { "label": "completed_group_5", "mean": 0.1109299436211586, "median": 0.10745325684547424, "p05": 0.07642066478729248, "p95": 0.15984633564949036 }, { "label": "current_group_partial", "mean": 0.6134276390075684, "median": 0.6200529336929321, "p05": 0.49638131260871887, "p95": 0.7104787826538086 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.629331111907959, "group": 6, "latest_source_mass": 0.45539987087249756, "layer": 24, "max_source_mass": 0.45539987087249756, "mixer_index": 46, "normalized_entropy": 0.837310557580963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14790503680706024, "median": 0.15420395135879517, "p05": 0.08257082104682922, "p95": 0.20396333932876587 }, { "label": "completed_group_1", "mean": 0.08658000826835632, "median": 0.08578156679868698, "p05": 0.07299856841564178, "p95": 0.10278668999671936 }, { "label": "completed_group_2", "mean": 0.06608415395021439, "median": 0.06551258265972137, "p05": 0.054121918976306915, "p95": 0.07912646979093552 }, { "label": "completed_group_3", "mean": 0.06427294760942459, "median": 0.06133077293634415, "p05": 0.04914242774248123, "p95": 0.09066928923130035 }, { "label": "completed_group_4", "mean": 0.0819322019815445, "median": 0.08114765584468842, "p05": 0.06886082887649536, "p95": 0.09821749478578568 }, { "label": "completed_group_5", "mean": 0.0978257954120636, "median": 0.09643188118934631, "p05": 0.07740232348442078, "p95": 0.12209523469209671 }, { "label": "current_group_partial", "mean": 0.45539987087249756, "median": 0.45274412631988525, "p05": 0.3780696392059326, "p95": 0.5462272763252258 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.338958501815796, "group": 6, "latest_source_mass": 0.5946891903877258, "layer": 24, "max_source_mass": 0.5946891903877258, "mixer_index": 47, "normalized_entropy": 0.6880885545850224, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10803166776895523, "median": 0.09507323801517487, "p05": 0.04510519653558731, "p95": 0.2335781455039978 }, { "label": "completed_group_1", "mean": 0.0559723861515522, "median": 0.05525296553969383, "p05": 0.042058296501636505, "p95": 0.07207933813333511 }, { "label": "completed_group_2", "mean": 0.047874726355075836, "median": 0.04723185673356056, "p05": 0.03294869884848595, "p95": 0.06497423350811005 }, { "label": "completed_group_3", "mean": 0.060293328016996384, "median": 0.05662515014410019, "p05": 0.040346790105104446, "p95": 0.08978720009326935 }, { "label": "completed_group_4", "mean": 0.029215168207883835, "median": 0.028496116399765015, "p05": 0.021991699934005737, "p95": 0.039112430065870285 }, { "label": "completed_group_5", "mean": 0.10392352938652039, "median": 0.10063231736421585, "p05": 0.06900570541620255, "p95": 0.1502695530653 }, { "label": "current_group_partial", "mean": 0.5946891903877258, "median": 0.600748598575592, "p05": 0.46449366211891174, "p95": 0.7016042470932007 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7611865997314453, "group": 7, "latest_source_mass": 0.3679775297641754, "layer": 25, "max_source_mass": 0.3679775297641754, "mixer_index": 48, "normalized_entropy": 0.9050708742058073, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1573912650346756, "median": 0.15898014605045319, "p05": 0.09474922716617584, "p95": 0.2298184633255005 }, { "label": "completed_group_1", "mean": 0.10317619889974594, "median": 0.10270150750875473, "p05": 0.08812867850065231, "p95": 0.11994096636772156 }, { "label": "completed_group_2", "mean": 0.08533225208520889, "median": 0.08495655655860901, "p05": 0.07036984711885452, "p95": 0.1013721376657486 }, { "label": "completed_group_3", "mean": 0.08218386024236679, "median": 0.07942138612270355, "p05": 0.06530377268791199, "p95": 0.1088358536362648 }, { "label": "completed_group_4", "mean": 0.09052872657775879, "median": 0.08931052684783936, "p05": 0.07639768719673157, "p95": 0.10804671049118042 }, { "label": "completed_group_5", "mean": 0.11341015249490738, "median": 0.1123356968164444, "p05": 0.09177497029304504, "p95": 0.13803066313266754 }, { "label": "completed_group_6", "mean": 0.3679775297641754, "median": 0.3671194612979889, "p05": 0.2961871325969696, "p95": 0.4462817311286926 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.476489782333374, "group": 7, "latest_source_mass": 0.3396714925765991, "layer": 25, "max_source_mass": 0.3836256265640259, "mixer_index": 49, "normalized_entropy": 0.7100414956318613, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09070298075675964, "median": 0.0818760022521019, "p05": 0.04576021432876587, "p95": 0.1613437533378601 }, { "label": "completed_group_1", "mean": 0.03713828697800636, "median": 0.035560622811317444, "p05": 0.02337636426091194, "p95": 0.054472144693136215 }, { "label": "completed_group_2", "mean": 0.030013620853424072, "median": 0.028701743111014366, "p05": 0.01836019568145275, "p95": 0.04582716524600983 }, { "label": "completed_group_3", "mean": 0.03533655032515526, "median": 0.031870774924755096, "p05": 0.018502909690141678, "p95": 0.061826568096876144 }, { "label": "completed_group_4", "mean": 0.020730752497911453, "median": 0.020491186529397964, "p05": 0.014567261561751366, "p95": 0.027856238186359406 }, { "label": "completed_group_5", "mean": 0.06278066337108612, "median": 0.059403590857982635, "p05": 0.03608490154147148, "p95": 0.10122650861740112 }, { "label": "completed_group_6", "mean": 0.3836256265640259, "median": 0.3773127794265747, "p05": 0.1874689906835556, "p95": 0.5895293951034546 }, { "label": "current_group_partial", "mean": 0.3396714925765991, "median": 0.34821513295173645, "p05": 0.09212851524353027, "p95": 0.587702751159668 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7213938236236572, "group": 7, "latest_source_mass": 0.37924307584762573, "layer": 26, "max_source_mass": 0.37924307584762573, "mixer_index": 50, "normalized_entropy": 0.8278154442529139, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10584281384944916, "median": 0.10982948541641235, "p05": 0.03868259862065315, "p95": 0.16580678522586823 }, { "label": "completed_group_1", "mean": 0.06887699663639069, "median": 0.06925956159830093, "p05": 0.04082746431231499, "p95": 0.09546782076358795 }, { "label": "completed_group_2", "mean": 0.05207539349794388, "median": 0.051871635019779205, "p05": 0.035383015871047974, "p95": 0.07048650085926056 }, { "label": "completed_group_3", "mean": 0.050678279250860214, "median": 0.04831986129283905, "p05": 0.03378130868077278, "p95": 0.07452918589115143 }, { "label": "completed_group_4", "mean": 0.04768219590187073, "median": 0.04721469432115555, "p05": 0.03680850565433502, "p95": 0.060343313962221146 }, { "label": "completed_group_5", "mean": 0.06809806823730469, "median": 0.06714263558387756, "p05": 0.04759174585342407, "p95": 0.09223967790603638 }, { "label": "completed_group_6", "mean": 0.2275032103061676, "median": 0.22425106167793274, "p05": 0.14320722222328186, "p95": 0.3203037679195404 }, { "label": "current_group_partial", "mean": 0.37924307584762573, "median": 0.3716055750846863, "p05": 0.20950253307819366, "p95": 0.5737906098365784 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5405172109603882, "group": 7, "latest_source_mass": 0.4017985463142395, "layer": 26, "max_source_mass": 0.4017985463142395, "mixer_index": 51, "normalized_entropy": 0.7408321802188831, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.074287548661232, "median": 0.06164507195353508, "p05": 0.028203874826431274, "p95": 0.19368377327919006 }, { "label": "completed_group_1", "mean": 0.04491765797138214, "median": 0.04365704953670502, "p05": 0.02776508405804634, "p95": 0.06526201963424683 }, { "label": "completed_group_2", "mean": 0.035157643258571625, "median": 0.03418130800127983, "p05": 0.022541508078575134, "p95": 0.050758615136146545 }, { "label": "completed_group_3", "mean": 0.046252522617578506, "median": 0.04353209584951401, "p05": 0.026215802878141403, "p95": 0.07329559326171875 }, { "label": "completed_group_4", "mean": 0.023427214473485947, "median": 0.023055266588926315, "p05": 0.016644639894366264, "p95": 0.03150089830160141 }, { "label": "completed_group_5", "mean": 0.07749318331480026, "median": 0.07363636791706085, "p05": 0.04229677468538284, "p95": 0.12579958140850067 }, { "label": "completed_group_6", "mean": 0.29666566848754883, "median": 0.2920765280723572, "p05": 0.14379237592220306, "p95": 0.45965293049812317 }, { "label": "current_group_partial", "mean": 0.4017985463142395, "median": 0.4004032611846924, "p05": 0.18456703424453735, "p95": 0.6275562047958374 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8063910007476807, "group": 7, "latest_source_mass": 0.3437207341194153, "layer": 27, "max_source_mass": 0.3437207341194153, "mixer_index": 52, "normalized_entropy": 0.868690446228377, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11433833092451096, "median": 0.11714513599872589, "p05": 0.06115472689270973, "p95": 0.16258974373340607 }, { "label": "completed_group_1", "mean": 0.07374332845211029, "median": 0.0731307864189148, "p05": 0.05447638779878616, "p95": 0.09509895741939545 }, { "label": "completed_group_2", "mean": 0.05932536721229553, "median": 0.05929289758205414, "p05": 0.04452420026063919, "p95": 0.07498160749673843 }, { "label": "completed_group_3", "mean": 0.05389238893985748, "median": 0.051720209419727325, "p05": 0.03926049917936325, "p95": 0.07536239922046661 }, { "label": "completed_group_4", "mean": 0.06126017868518829, "median": 0.06108957529067993, "p05": 0.0506296306848526, "p95": 0.07238088548183441 }, { "label": "completed_group_5", "mean": 0.07349519431591034, "median": 0.07249070703983307, "p05": 0.05706139653921127, "p95": 0.09332700818777084 }, { "label": "completed_group_6", "mean": 0.22022445499897003, "median": 0.21959853172302246, "p05": 0.15368635952472687, "p95": 0.2904376685619354 }, { "label": "current_group_partial", "mean": 0.3437207341194153, "median": 0.33986932039260864, "p05": 0.22015926241874695, "p95": 0.4802429676055908 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.4859801530838013, "group": 7, "latest_source_mass": 0.4424910843372345, "layer": 27, "max_source_mass": 0.4424910843372345, "mixer_index": 53, "normalized_entropy": 0.7146053992378076, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08387120068073273, "median": 0.07644708454608917, "p05": 0.0386628657579422, "p95": 0.15876422822475433 }, { "label": "completed_group_1", "mean": 0.041406162083148956, "median": 0.0394824743270874, "p05": 0.025755178183317184, "p95": 0.06136181950569153 }, { "label": "completed_group_2", "mean": 0.032444946467876434, "median": 0.0307333804666996, "p05": 0.018246563151478767, "p95": 0.052875272929668427 }, { "label": "completed_group_3", "mean": 0.03686721250414848, "median": 0.03247762471437454, "p05": 0.017809681594371796, "p95": 0.06780916452407837 }, { "label": "completed_group_4", "mean": 0.025052767246961594, "median": 0.024561133235692978, "p05": 0.017057543620467186, "p95": 0.034822143614292145 }, { "label": "completed_group_5", "mean": 0.06012478470802307, "median": 0.056938059628009796, "p05": 0.03254516050219536, "p95": 0.09905251860618591 }, { "label": "completed_group_6", "mean": 0.27774184942245483, "median": 0.2747848629951477, "p05": 0.11654777824878693, "p95": 0.4562506675720215 }, { "label": "current_group_partial", "mean": 0.4424910843372345, "median": 0.4387660026550293, "p05": 0.21194897592067719, "p95": 0.6782112717628479 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7777256965637207, "group": 7, "latest_source_mass": 0.3409850299358368, "layer": 28, "max_source_mass": 0.3409850299358368, "mixer_index": 54, "normalized_entropy": 0.8549053488311195, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10586117208003998, "median": 0.10652337223291397, "p05": 0.05095779523253441, "p95": 0.1629783660173416 }, { "label": "completed_group_1", "mean": 0.07087038457393646, "median": 0.06906798481941223, "p05": 0.04985840618610382, "p95": 0.09773125499486923 }, { "label": "completed_group_2", "mean": 0.05850498005747795, "median": 0.05767729878425598, "p05": 0.04179685190320015, "p95": 0.07779045403003693 }, { "label": "completed_group_3", "mean": 0.06360965222120285, "median": 0.06068425625562668, "p05": 0.04235687851905823, "p95": 0.09371945261955261 }, { "label": "completed_group_4", "mean": 0.04672512412071228, "median": 0.04617916792631149, "p05": 0.0356140062212944, "p95": 0.05914818495512009 }, { "label": "completed_group_5", "mean": 0.07485505938529968, "median": 0.07274073362350464, "p05": 0.052147433161735535, "p95": 0.10333258658647537 }, { "label": "completed_group_6", "mean": 0.2385886013507843, "median": 0.2345724105834961, "p05": 0.15389031171798706, "p95": 0.33735349774360657 }, { "label": "current_group_partial", "mean": 0.3409850299358368, "median": 0.33667099475860596, "p05": 0.18655306100845337, "p95": 0.5089476704597473 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6318418979644775, "group": 7, "latest_source_mass": 0.3292362689971924, "layer": 28, "max_source_mass": 0.3292362689971924, "mixer_index": 55, "normalized_entropy": 0.7847500712360619, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09244748950004578, "median": 0.06782519817352295, "p05": 0.025312144309282303, "p95": 0.24087309837341309 }, { "label": "completed_group_1", "mean": 0.05762210115790367, "median": 0.056058838963508606, "p05": 0.03864594176411629, "p95": 0.08028930425643921 }, { "label": "completed_group_2", "mean": 0.045099519193172455, "median": 0.04426075518131256, "p05": 0.03003283590078354, "p95": 0.06244165822863579 }, { "label": "completed_group_3", "mean": 0.056236449629068375, "median": 0.052902162075042725, "p05": 0.03282861411571503, "p95": 0.08620619773864746 }, { "label": "completed_group_4", "mean": 0.029373466968536377, "median": 0.02887408435344696, "p05": 0.020401213318109512, "p95": 0.04018516466021538 }, { "label": "completed_group_5", "mean": 0.07237439602613449, "median": 0.06993190199136734, "p05": 0.04555661231279373, "p95": 0.10748817771673203 }, { "label": "completed_group_6", "mean": 0.31761035323143005, "median": 0.3173876404762268, "p05": 0.16091161966323853, "p95": 0.4724525213241577 }, { "label": "current_group_partial", "mean": 0.3292362689971924, "median": 0.3128150701522827, "p05": 0.12605780363082886, "p95": 0.5847846865653992 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8334553241729736, "group": 8, "latest_source_mass": 0.270294725894928, "layer": 29, "max_source_mass": 0.27155569195747375, "mixer_index": 56, "normalized_entropy": 0.8817056346252721, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1157165914773941, "median": 0.113271564245224, "p05": 0.06316040456295013, "p95": 0.17676270008087158 }, { "label": "completed_group_1", "mean": 0.07161639630794525, "median": 0.07118864357471466, "p05": 0.04294680058956146, "p95": 0.102086141705513 }, { "label": "completed_group_2", "mean": 0.06577154994010925, "median": 0.06490545719861984, "p05": 0.04486069828271866, "p95": 0.09022088348865509 }, { "label": "completed_group_3", "mean": 0.07425849884748459, "median": 0.07179903984069824, "p05": 0.045271825045347214, "p95": 0.11117851734161377 }, { "label": "completed_group_4", "mean": 0.04881400242447853, "median": 0.04833249747753143, "p05": 0.0363147035241127, "p95": 0.0629924014210701 }, { "label": "completed_group_5", "mean": 0.08197253197431564, "median": 0.08029600977897644, "p05": 0.05680408701300621, "p95": 0.11223416775465012 }, { "label": "completed_group_6", "mean": 0.27155569195747375, "median": 0.2687028646469116, "p05": 0.18294592201709747, "p95": 0.3740781545639038 }, { "label": "completed_group_7", "mean": 0.270294725894928, "median": 0.2569413185119629, "p05": 0.13936233520507812, "p95": 0.43686625361442566 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6227548122406006, "group": 8, "latest_source_mass": 0.45173269510269165, "layer": 29, "max_source_mass": 0.45173269510269165, "mixer_index": 57, "normalized_entropy": 0.7385475426494315, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0605507455766201, "median": 0.05293343961238861, "p05": 0.02206447720527649, "p95": 0.1254502236843109 }, { "label": "completed_group_1", "mean": 0.040063515305519104, "median": 0.03803131729364395, "p05": 0.022060411050915718, "p95": 0.06445677578449249 }, { "label": "completed_group_2", "mean": 0.032619886100292206, "median": 0.030926726758480072, "p05": 0.020269516855478287, "p95": 0.050711989402770996 }, { "label": "completed_group_3", "mean": 0.03868011012673378, "median": 0.0344192236661911, "p05": 0.020861119031906128, "p95": 0.06798171997070312 }, { "label": "completed_group_4", "mean": 0.020290488377213478, "median": 0.01974661275744438, "p05": 0.014590941369533539, "p95": 0.027847949415445328 }, { "label": "completed_group_5", "mean": 0.04760073497891426, "median": 0.044880062341690063, "p05": 0.027415042743086815, "p95": 0.07675685733556747 }, { "label": "completed_group_6", "mean": 0.19551873207092285, "median": 0.18908299505710602, "p05": 0.10506269335746765, "p95": 0.30520933866500854 }, { "label": "completed_group_7", "mean": 0.11294309794902802, "median": 0.09412655979394913, "p05": 0.03902517631649971, "p95": 0.24146248400211334 }, { "label": "current_group_partial", "mean": 0.45173269510269165, "median": 0.4618796110153198, "p05": 0.24265648424625397, "p95": 0.6204620599746704 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9787883758544922, "group": 8, "latest_source_mass": 0.1984800100326538, "layer": 30, "max_source_mass": 0.24502354860305786, "mixer_index": 58, "normalized_entropy": 0.9005854004479842, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0788881778717041, "median": 0.055554650723934174, "p05": 0.03367501497268677, "p95": 0.21530622243881226 }, { "label": "completed_group_1", "mean": 0.08108963817358017, "median": 0.07848536968231201, "p05": 0.05139587074518204, "p95": 0.12043403089046478 }, { "label": "completed_group_2", "mean": 0.057054273784160614, "median": 0.056182146072387695, "p05": 0.04127398133277893, "p95": 0.07570968568325043 }, { "label": "completed_group_3", "mean": 0.08211587369441986, "median": 0.0802752673625946, "p05": 0.055154237896203995, "p95": 0.113825723528862 }, { "label": "completed_group_4", "mean": 0.026040805503726006, "median": 0.025504354387521744, "p05": 0.01917124353349209, "p95": 0.03502596169710159 }, { "label": "completed_group_5", "mean": 0.10537257045507431, "median": 0.10299044847488403, "p05": 0.07248030602931976, "p95": 0.14682361483573914 }, { "label": "completed_group_6", "mean": 0.24502354860305786, "median": 0.24406442046165466, "p05": 0.1495005488395691, "p95": 0.3413054645061493 }, { "label": "completed_group_7", "mean": 0.12593510746955872, "median": 0.11572076380252838, "p05": 0.051278628408908844, "p95": 0.2299422323703766 }, { "label": "current_group_partial", "mean": 0.1984800100326538, "median": 0.19106052815914154, "p05": 0.11614266037940979, "p95": 0.3049595355987549 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7590525150299072, "group": 8, "latest_source_mass": 0.3258843719959259, "layer": 30, "max_source_mass": 0.3258843719959259, "mixer_index": 59, "normalized_entropy": 0.800579300438408, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07131639868021011, "median": 0.052984386682510376, "p05": 0.03260878100991249, "p95": 0.19837424159049988 }, { "label": "completed_group_1", "mean": 0.044687941670417786, "median": 0.04335416108369827, "p05": 0.027018578723073006, "p95": 0.06556333601474762 }, { "label": "completed_group_2", "mean": 0.033290669322013855, "median": 0.03222261369228363, "p05": 0.01970716379582882, "p95": 0.05042369291186333 }, { "label": "completed_group_3", "mean": 0.043597716838121414, "median": 0.03997292369604111, "p05": 0.024005703628063202, "p95": 0.07244924455881119 }, { "label": "completed_group_4", "mean": 0.02658817172050476, "median": 0.02605675905942917, "p05": 0.017710059881210327, "p95": 0.03680335357785225 }, { "label": "completed_group_5", "mean": 0.06922741234302521, "median": 0.06634850800037384, "p05": 0.03838749974966049, "p95": 0.10990548133850098 }, { "label": "completed_group_6", "mean": 0.23554196953773499, "median": 0.2292930632829666, "p05": 0.10460641980171204, "p95": 0.3925905227661133 }, { "label": "completed_group_7", "mean": 0.14986534416675568, "median": 0.12543796002864838, "p05": 0.04750998318195343, "p95": 0.3270293176174164 }, { "label": "current_group_partial", "mean": 0.3258843719959259, "median": 0.31322309374809265, "p05": 0.1389845609664917, "p95": 0.5583901405334473 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9736928939819336, "group": 8, "latest_source_mass": 0.20976582169532776, "layer": 31, "max_source_mass": 0.22560220956802368, "mixer_index": 60, "normalized_entropy": 0.8982663467084998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09849467873573303, "median": 0.09779177606105804, "p05": 0.017307927832007408, "p95": 0.19556622207164764 }, { "label": "completed_group_1", "mean": 0.068942591547966, "median": 0.0661388635635376, "p05": 0.044743385165929794, "p95": 0.10368303209543228 }, { "label": "completed_group_2", "mean": 0.0599900484085083, "median": 0.05786054581403732, "p05": 0.040265996009111404, "p95": 0.08643331378698349 }, { "label": "completed_group_3", "mean": 0.06335455179214478, "median": 0.05964011698961258, "p05": 0.04099462926387787, "p95": 0.09641734510660172 }, { "label": "completed_group_4", "mean": 0.04671281576156616, "median": 0.04506188631057739, "p05": 0.03403949737548828, "p95": 0.06450185924768448 }, { "label": "completed_group_5", "mean": 0.0713595524430275, "median": 0.06962447613477707, "p05": 0.048271458595991135, "p95": 0.10105480253696442 }, { "label": "completed_group_6", "mean": 0.22560220956802368, "median": 0.22400957345962524, "p05": 0.13957585394382477, "p95": 0.31992682814598083 }, { "label": "completed_group_7", "mean": 0.1557777225971222, "median": 0.1399117112159729, "p05": 0.06281191110610962, "p95": 0.2967374622821808 }, { "label": "current_group_partial", "mean": 0.20976582169532776, "median": 0.2105630338191986, "p05": 0.08139262348413467, "p95": 0.34620192646980286 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.809161901473999, "group": 8, "latest_source_mass": 0.288146048784256, "layer": 31, "max_source_mass": 0.288146048784256, "mixer_index": 61, "normalized_entropy": 0.8233850650202157, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08854076266288757, "median": 0.08406365662813187, "p05": 0.020090684294700623, "p95": 0.1807844340801239 }, { "label": "completed_group_1", "mean": 0.05356752499938011, "median": 0.05036265030503273, "p05": 0.031541675329208374, "p95": 0.08521421998739243 }, { "label": "completed_group_2", "mean": 0.04243364930152893, "median": 0.03939494490623474, "p05": 0.023794323205947876, "p95": 0.0721740797162056 }, { "label": "completed_group_3", "mean": 0.04702243581414223, "median": 0.043501369655132294, "p05": 0.02430840954184532, "p95": 0.0788305327296257 }, { "label": "completed_group_4", "mean": 0.02648795023560524, "median": 0.025087328627705574, "p05": 0.01698113977909088, "p95": 0.040197182446718216 }, { "label": "completed_group_5", "mean": 0.05820769816637039, "median": 0.05441046878695488, "p05": 0.02834305912256241, "p95": 0.10125020891427994 }, { "label": "completed_group_6", "mean": 0.18489578366279602, "median": 0.17386877536773682, "p05": 0.07876831293106079, "p95": 0.32456493377685547 }, { "label": "completed_group_7", "mean": 0.21069815754890442, "median": 0.19004324078559875, "p05": 0.07121473550796509, "p95": 0.42417171597480774 }, { "label": "current_group_partial", "mean": 0.288146048784256, "median": 0.28367581963539124, "p05": 0.0962119847536087, "p95": 0.5029257535934448 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.8544633388519287, "group": 8, "latest_source_mass": 0.12106549739837646, "layer": 32, "max_source_mass": 0.3525030016899109, "mixer_index": 62, "normalized_entropy": 0.8440026376822011, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.07977832853794098, "median": 0.053830958902835846, "p05": 0.02079753391444683, "p95": 0.2620829939842224 }, { "label": "completed_group_1", "mean": 0.05858254432678223, "median": 0.056158941239118576, "p05": 0.03545547276735306, "p95": 0.09179478138685226 }, { "label": "completed_group_2", "mean": 0.053744811564683914, "median": 0.05185101926326752, "p05": 0.03345105051994324, "p95": 0.08065605163574219 }, { "label": "completed_group_3", "mean": 0.07626347988843918, "median": 0.07124648243188858, "p05": 0.043040163815021515, "p95": 0.12173676490783691 }, { "label": "completed_group_4", "mean": 0.031090255826711655, "median": 0.02958102896809578, "p05": 0.01873353309929371, "p95": 0.04807420074939728 }, { "label": "completed_group_5", "mean": 0.0994817242026329, "median": 0.0973888486623764, "p05": 0.0575672909617424, "p95": 0.14856761693954468 }, { "label": "completed_group_6", "mean": 0.3525030016899109, "median": 0.3572760224342346, "p05": 0.18325307965278625, "p95": 0.5024369955062866 }, { "label": "completed_group_7", "mean": 0.1274903416633606, "median": 0.1073581799864769, "p05": 0.039848171174526215, "p95": 0.2741084098815918 }, { "label": "current_group_partial", "mean": 0.12106549739837646, "median": 0.10882791876792908, "p05": 0.0503288172185421, "p95": 0.22616314888000488 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9456019401550293, "group": 8, "latest_source_mass": 0.18013209104537964, "layer": 32, "max_source_mass": 0.2524128556251526, "mixer_index": 63, "normalized_entropy": 0.885481602665194, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08823961019515991, "median": 0.07105386257171631, "p05": 0.04482517018914223, "p95": 0.1774362027645111 }, { "label": "completed_group_1", "mean": 0.06737187504768372, "median": 0.06628458946943283, "p05": 0.04582075774669647, "p95": 0.09290395677089691 }, { "label": "completed_group_2", "mean": 0.05197303742170334, "median": 0.051264435052871704, "p05": 0.035227708518505096, "p95": 0.07133325934410095 }, { "label": "completed_group_3", "mean": 0.0631948709487915, "median": 0.06016676500439644, "p05": 0.03969947248697281, "p95": 0.09776238352060318 }, { "label": "completed_group_4", "mean": 0.02739548496901989, "median": 0.026907969266176224, "p05": 0.020028207451105118, "p95": 0.036658842116594315 }, { "label": "completed_group_5", "mean": 0.08291903138160706, "median": 0.08066994696855545, "p05": 0.05307728424668312, "p95": 0.12080682814121246 }, { "label": "completed_group_6", "mean": 0.2524128556251526, "median": 0.25180506706237793, "p05": 0.14340050518512726, "p95": 0.36010295152664185 }, { "label": "completed_group_7", "mean": 0.186361163854599, "median": 0.17697101831436157, "p05": 0.08843781799077988, "p95": 0.3156980276107788 }, { "label": "current_group_partial", "mean": 0.18013209104537964, "median": 0.16958200931549072, "p05": 0.08391770720481873, "p95": 0.3105517029762268 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.596132755279541, "group": null, "latest_source_mass": 0.29075440764427185, "layer": null, "max_source_mass": 0.3670029938220978, "mixer_index": null, "normalized_entropy": 0.7264313223797062, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09668220579624176, "median": 0.1013428270816803, "p05": 0.02658359333872795, "p95": 0.15407970547676086 }, { "label": "completed_group_1", "mean": 0.037084318697452545, "median": 0.035886794328689575, "p05": 0.02348068356513977, "p95": 0.053765345364809036 }, { "label": "completed_group_2", "mean": 0.03457995876669884, "median": 0.03355112299323082, "p05": 0.019709739834070206, "p95": 0.0531856045126915 }, { "label": "completed_group_3", "mean": 0.026012206450104713, "median": 0.023950476199388504, "p05": 0.013590577989816666, "p95": 0.04525115340948105 }, { "label": "completed_group_4", "mean": 0.0258483849465847, "median": 0.02512776106595993, "p05": 0.016705092042684555, "p95": 0.03687155246734619 }, { "label": "completed_group_5", "mean": 0.037477150559425354, "median": 0.03506678715348244, "p05": 0.02071693353354931, "p95": 0.061963990330696106 }, { "label": "completed_group_6", "mean": 0.0845583900809288, "median": 0.07240844517946243, "p05": 0.03393660485744476, "p95": 0.1794772744178772 }, { "label": "completed_group_7", "mean": 0.3670029938220978, "median": 0.3608853220939636, "p05": 0.1103469580411911, "p95": 0.6465575098991394 }, { "label": "completed_group_8", "mean": 0.29075440764427185, "median": 0.28802984952926636, "p05": 0.039541684091091156, "p95": 0.5658512711524963 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "optimizer_state_before": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.09253507107496262, 0.19514523446559906, 0.07917427271604538, 0.04215213656425476, 0.12552575767040253, 0.05027730390429497, 0.054851483553647995, 0.036243390291929245, 0.031880609691143036, 0.035054463893175125, 0.050445348024368286, 0.05600574240088463, 0.053015924990177155, 0.032114963978528976, 0.048427484929561615, 0.5742782950401306, 0.04075472429394722, 0.03505101054906845, 0.06577442586421967, 0.08553697913885117, 0.0493597537279129, 0.023477228358387947, 0.03512806445360184, 0.030729109421372414, 0.023035654798150063, 0.019568851217627525, 0.02376176416873932, 0.02176014520227909, 0.03424545377492905, 0.021422436460852623, 0.019477492198348045, 0.017491821199655533 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1261602139711613e-05, "first_to_last_ratio": 0.5245701598350286, "imbalance_abs_log_ratio": 0.6451760949151356, "last_quartile_mean": 2.146824772353284e-05, "mean": 2.0086123484475138e-05, "normalized": [ 0.6085748623432731, 0.672343922700359, 0.5609816766389839, 0.5375859907162287, 0.641556317168069, 0.5340725535941322, 0.4789883309018991, 0.4512226054976404, 0.5900913956594765, 0.540676107431406, 0.5086908977229898, 0.473500107930304, 0.40487760145311497, 0.38162857664898325, 0.3654861410934707, 0.44670015436893523, 1.1143309981822054, 1.1164864049647665, 1.036008527266956, 0.9253333108915559, 3.7714016683827163, 3.048770143020733, 2.3679155285839633, 1.872296940712075, 1.7902480367185682, 1.5773203624246581, 1.2887757729919698, 0.9509073151516803, 1.1740622366025057, 0.7631942144418731, 0.5687018744547615, 0.43726942333974633 ], "peak_layer": 21, "peak_normalized": 3.7714016683827163, "population_cv": 0.7926339194277571, "rest_mean": 1.4245798663776239e-05, "spike_contrast": 3.6237966529399817, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.1623877516249196e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.2223909834574442e-05, 1.350478305539582e-05, 1.1267947229498532e-05, 1.0798018593050074e-05, 1.2886379408882931e-05, 1.0727447261160705e-05, 9.621018762118183e-06, 9.063312973012216e-06, 1.1852648640342522e-05, 1.0860087058972567e-05, 1.0217628187092487e-05, 9.510781637800392e-06, 8.132421498885378e-06, 7.665438715775963e-06, 7.341199761867756e-06, 8.97247446118854e-06, 2.238259003206622e-05, 2.2425883798860013e-05, 2.0809395209653303e-05, 1.8586359146866016e-05, 7.575283962069079e-05, 6.123797356849536e-05, 4.756224370794371e-05, 3.760718755074777e-05, 3.595914313336834e-05, 3.168225157423876e-05, 2.5886509320116602e-05, 1.9100041754427366e-05, 2.3582359062856995e-05, 1.532961323391646e-05, 1.1423016076150816e-05, 8.78304763318738e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.418448133947095e-05, "first_to_last_ratio": 0.515514370401758, "imbalance_abs_log_ratio": 0.662590099285622, "last_quartile_mean": 8.570950467401417e-05, "mean": 8.019432914352365e-05, "normalized": [ 0.5977618675800617, 0.6560769396425044, 0.5549086183167843, 0.5332641078022274, 0.6187504975834317, 0.5235180355140238, 0.4739614950807707, 0.44949966520899015, 0.5901273331445138, 0.5434806557289369, 0.5105162863240122, 0.47519844997761185, 0.40525254510695746, 0.3803726356397213, 0.3632907544081655, 0.4413534187685878, 1.1120243122062796, 1.1101422281940985, 1.048076642833664, 0.9240879167666287, 3.7925106998539326, 3.0713745688935163, 2.3834694147362985, 1.8907998104142179, 1.7924928574135153, 1.575834720390749, 1.2856009256033192, 0.946613406637061, 1.179458790809137, 0.7622280566154953, 0.5700461300235112, 0.43790621278127495 ], "peak_layer": 21, "peak_normalized": 3.7925106998539326, "population_cv": 0.799556889668146, "rest_mean": 5.663903492084636e-05, "spike_contrast": 3.6616605179769537, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020739291794598104, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 4.793711195816286e-05, 5.261365004116669e-05, 4.450052438187413e-05, 4.27647573815193e-05, 4.9620281060924754e-05, 4.1983177652582526e-05, 3.800902413786389e-05, 3.604732410167344e-05, 4.7324865590780973e-05, 4.358406658866443e-05, 4.09405110985972e-05, 3.810822090599686e-05, 3.249895598855801e-05, 3.0503728339681402e-05, 2.913385833380744e-05, 3.539404133334756e-05, 8.917804370867088e-05, 8.902711124392226e-05, 8.404980326304212e-05, 7.410661055473611e-05, 0.0003041378513444215, 0.0002463068231008947, 0.00019114073074888438, 0.0001516314223408699, 0.00014374776219483465, 0.00012637300824280828, 0.00010309790377505124, 7.591302710352466e-05, 9.458590648137033e-05, 6.11263676546514e-05, 4.5714466978097335e-05, 3.511759496177547e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.039183240849524736, "first_to_last_ratio": 0.5155143831205776, "imbalance_abs_log_ratio": 0.6625900746135269, "last_quartile_mean": 0.07600804581306875, "mean": 0.07111713383346796, "normalized": [ 0.5977618468462375, 0.6560769852418055, 0.5549086482011255, 0.5332641116607529, 0.618750502626057, 0.5235180409966333, 0.4739615115435027, 0.44949963686515554, 0.590127313076766, 0.543480629182993, 0.5105162957833653, 0.47519841853167755, 0.40525256719553, 0.38037265430860584, 0.36329073465232276, 0.44135337886041204, 1.1120242741359478, 1.1101421724087162, 1.048076711910138, 0.9240879569048468, 3.79251093493639, 3.0713746221735128, 2.383469321905541, 1.8907997296698473, 1.7924929415619593, 1.5758345890827283, 1.2856009880693582, 0.9466133568642321, 1.179458727081905, 0.7622280464822554, 0.5700461310047829, 0.4379062202348958 ], "peak_layer": 21, "peak_normalized": 3.79251093493639, "population_cv": 0.7995569122050559, "rest_mean": 0.05022806260320875, "spike_contrast": 3.6616606125102327, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.18391811847686768, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.04251110926270485, 0.046658314764499664, 0.03946351259946823, 0.03792421519756317, 0.04400376230478287, 0.03723110258579254, 0.03370678424835205, 0.031967125833034515, 0.04196816310286522, 0.03865078464150429, 0.03630645573139191, 0.03379474952816963, 0.028820401057600975, 0.027051012963056564, 0.025836195796728134, 0.03138778731226921, 0.07908397912979126, 0.07895012944936752, 0.07453621178865433, 0.06571848690509796, 0.26971250772476196, 0.2184273600578308, 0.1695055067539215, 0.13446825742721558, 0.12747696042060852, 0.11206883937120438, 0.0914282575249672, 0.06732042878866196, 0.08387972414493561, 0.05420747399330139, 0.040540046989917755, 0.03114263527095318 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8925995988183786e-06, "first_to_last_ratio": 0.3261648413799994, "imbalance_abs_log_ratio": 1.1203523770075996, "last_quartile_mean": 5.802586173331292e-06, "mean": 4.670730803013612e-06, "normalized": [ 0.5310348802393875, 0.46146348274830745, 0.44476201867725484, 0.43677383492791344, 0.3556033019654118, 0.3423979278935454, 0.3333438108043655, 0.33625410358276536, 0.4223966230117983, 0.41208725398123947, 0.41437801392793316, 0.4155995059181246, 0.382562022554051, 0.3815658727614762, 0.38949800087449105, 0.4096468013886889, 0.8179581009736591, 0.8423033340072443, 0.8553008421936446, 0.8530642636826707, 3.896213232744094, 3.3402464703409684, 2.714333567686501, 2.2725775720601913, 2.07067894660348, 1.8582090508366296, 1.5332685227055798, 1.1680391073619418, 1.334393737375711, 0.8510160788430678, 0.6469677209502473, 0.4760619963776144 ], "peak_layer": 21, "peak_normalized": 3.896213232744094, "population_cv": 0.9180343092487918, "rest_mean": 3.0629528535226e-06, "spike_contrast": 4.359431035612757, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3352731730265078e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.4803209726087516e-06, 2.15537170333846e-06, 2.0773636606463697e-06, 2.040053004748188e-06, 1.6609272961431998e-06, 1.599248548700416e-06, 1.5569592051178915e-06, 1.570552399243752e-06, 1.9729009181901347e-06, 1.9247486306994688e-06, 1.935448153744801e-06, 1.9411534140090225e-06, 1.786844222806394e-06, 1.7821914752857992e-06, 1.819240310396708e-06, 1.9133499336021487e-06, 3.820462097792188e-06, 3.934172127628699e-06, 3.99487998947734e-06, 3.984433533332776e-06, 1.8198163161287084e-05, 1.5601392078679055e-05, 1.2677921404247172e-05, 1.0614598068059422e-05, 9.671583939052653e-06, 8.679194252181333e-06, 7.161484518292127e-06, 5.455596237879945e-06, 6.2325939325091895e-06, 3.974867013312178e-06, 3.021812062797835e-06, 2.2235574306250783e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.4072450716848834e-05, "first_to_last_ratio": 0.634285224932807, "imbalance_abs_log_ratio": 0.4552565441138088, "last_quartile_mean": 5.371786915020493e-05, "mean": 5.588273802459298e-05, "normalized": [ 0.6676093152090673, 0.7197771199409104, 0.6218041460469672, 0.5988669281463308, 0.6666598935307795, 0.5782912860000531, 0.5252896450390306, 0.4994083671227573, 0.6402001675671648, 0.5894906078705415, 0.5499130564119773, 0.5108819562402718, 0.4646654187577452, 0.4356758329586985, 0.4154501696208735, 0.4880908883442812, 1.1821250357964774, 1.1664741605550812, 1.078174429062179, 0.962548943315326, 3.8682876465312988, 3.0459843127744204, 2.2760577554709505, 1.7581888525052323, 1.7091478115698895, 1.4520832989438006, 1.1543347914386743, 0.8578423927708055, 0.9984708567512235, 0.639080834302184, 0.48627901888802055, 0.39284506051698587 ], "peak_layer": 21, "peak_normalized": 3.8682876465312988, "population_cv": 0.7787255777921933, "rest_mean": 4.0033428242033415e-05, "spike_contrast": 3.533772076553717, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014146901085041464, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.7307836464606225e-05, 4.022311622975394e-05, 3.4748118196148425e-05, 3.346632365719415e-05, 3.72547801816836e-05, 3.231650043744594e-05, 2.935462362074759e-05, 2.7908306947210804e-05, 3.57761382474564e-05, 3.294234920758754e-05, 3.073064726777375e-05, 2.8549482522066683e-05, 2.596677586552687e-05, 2.434675843687728e-05, 2.321649299119599e-05, 2.727585524553433e-05, 6.606038368772715e-05, 6.518576992675662e-05, 6.025133916409686e-05, 5.3789870435139164e-05, 0.0002161705051548779, 0.00017021794337779284, 0.00012719233927782625, 9.825240704230964e-05, 9.551185939926654e-05, 8.114639058476314e-05, 6.450738874264061e-05, 4.7938581701600924e-05, 5.579728531301953e-05, 3.571358683984727e-05, 2.7174603019375354e-05, 2.1953257601126097e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.604143833195849e-05, "first_to_last_ratio": 0.864500772440069, "imbalance_abs_log_ratio": 0.14560308034841146, "last_quartile_mean": 3.012309434780036e-05, "mean": 3.750934104118642e-05, "normalized": [ 0.7613938466965254, 0.8106407342403823, 0.7210553143827715, 0.6926649057839166, 0.7441900072557481, 0.6549135446116822, 0.6010420947141472, 0.5682228952686117, 0.7146883031764985, 0.6605877661848203, 0.6100617443729247, 0.5720177685677394, 0.5632312795246776, 0.5253333685644455, 0.49762299559665957, 0.5644048416919647, 1.293995056666398, 1.2838583891993025, 1.1578542801495904, 1.0246076437213654, 3.9830246051038323, 3.000750743357827, 2.06181294387121, 1.5073651467427196, 1.5060057544008438, 1.2137998314477398, 0.951522556430117, 0.7372102220186553, 0.7702443482214394, 0.5110189167704164, 0.405035438834493, 0.3298227124305357 ], "peak_layer": 21, "peak_normalized": 3.9830246051038323, "population_cv": 0.7527493617080463, "rest_mean": 2.770278890104102e-05, "spike_contrast": 3.2655456792139823, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.046472259797156e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.855938146240078e-05, 3.0406599762500264e-05, 2.7046309696743265e-05, 2.5981404178310186e-05, 2.791407678159885e-05, 2.4565375497331843e-05, 2.2544692910742015e-05, 2.1313666366040707e-05, 2.6807487301994115e-05, 2.4778211809461936e-05, 2.2883014025865123e-05, 2.1456009562825784e-05, 2.1126434148754925e-05, 1.9704908481799066e-05, 1.866551065177191e-05, 2.1170453692320734e-05, 4.853690188610926e-05, 4.815668216906488e-05, 4.343035107012838e-05, 3.8432357541751117e-05, 0.0001494006282882765, 0.00011255618301220238, 7.733724487479776e-05, 5.654027336277068e-05, 5.648928345181048e-05, 4.5528831833507866e-05, 3.5690984077518806e-05, 2.7652269636746496e-05, 2.889135794248432e-05, 1.9167982827639207e-05, 1.5192612409009598e-05, 1.2371432603686117e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.932713899412192e-05, "first_to_last_ratio": 0.48428070567546516, "imbalance_abs_log_ratio": 0.7250905699794873, "last_quartile_mean": 0.00018445322712068446, "mean": 0.00016839920272104791, "normalized": [ 0.580113378070693, 0.6316011322538936, 0.5371709395526841, 0.5193916381660382, 0.5958096458866964, 0.49825289372245124, 0.4493748616089108, 0.43187515015476213, 0.5789448986804199, 0.5298739922879939, 0.5012758869564349, 0.4661125381937263, 0.3655735366227801, 0.35379219756409647, 0.33290305767004397, 0.4094948207474536, 1.0315615517644154, 1.032222526387711, 0.9835380166521113, 0.8863325773824939, 3.853305598042274, 3.162866144945307, 2.502991043300759, 2.002956870927857, 1.8436650111759416, 1.637369378572084, 1.333674654700857, 1.0052835546379626, 1.1909970302670063, 0.7575881320943505, 0.5664246618886806, 0.4276626791211106 ], "peak_layer": 21, "peak_normalized": 3.853305598042274, "population_cv": 0.8344872340072493, "rest_mean": 0.00011622174093241079, "spike_contrast": 3.873264096443705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004501574963796884, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.769063035491854e-05, 0.00010636112710926682, 9.045915794558823e-05, 8.746513776713982e-05, 0.00010033386934082955, 8.390539005631581e-05, 7.567436841782182e-05, 7.27274309610948e-05, 9.749385935720056e-05, 8.923035784391686e-05, 8.441445970674977e-05, 7.84929798101075e-05, 6.156229210318998e-05, 5.9578323998721316e-05, 5.606060949503444e-05, 6.895860133226961e-05, 0.00017371414287481457, 0.00017382545047439635, 0.0001656270178500563, 0.00014925769937690347, 0.0006488935905508697, 0.000532624137122184, 0.00042150169610977173, 0.000337296340148896, 0.00031047171796672046, 0.0002757316979113966, 0.0002245897485408932, 0.0001692889491096139, 0.00020056295034009963, 0.00012757723743561655, 9.538546146359295e-05, 7.201805419754237e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.116500623524189, 0.0468219518661499, 0.09375449270009995, 0.041188232600688934, 0.024237047880887985, 0.022429833188652992, 0.03027583472430706, 0.04546776041388512, 0.030174262821674347, 0.03663193807005882, 0.05748353153467178, 0.10596779733896255, 0.05394246056675911, 0.06537146866321564, 0.08060523867607117, 0.06690707802772522, 0.033449217677116394, 0.022565271705389023, 0.036697570234537125, 0.07367803901433945, 0.042231619358062744, 0.052135858684778214, 0.06269875913858414, 0.0728035643696785, 0.051585447043180466, 0.04862172156572342, 0.05972515419125557, 0.06787826865911484, 0.06439156830310822, 0.07232685387134552, 0.09627991914749146, 0.1271906942129135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1220664760003274e-05, "first_to_last_ratio": 0.6555362798323586, "imbalance_abs_log_ratio": 0.42230163050239594, "last_quartile_mean": 1.711677157345548e-05, "mean": 1.7638031224009865e-05, "normalized": [ 0.6920470508787427, 0.765871148622978, 0.6390872031764239, 0.6123477040772916, 0.7313980706371319, 0.6060462235938255, 0.5358729377549785, 0.5066345241410947, 0.6361374094466793, 0.5935131324046057, 0.553200745756426, 0.5203044559127653, 0.44270176965120756, 0.41555808291178226, 0.42011495887552464, 0.5249791307747981, 1.2555818719274776, 1.2709584825941431, 1.1358536396128, 1.0637844334884017, 3.6056946953031805, 2.7844654381945673, 2.1779734918269082, 1.746298720031033, 1.816337422755615, 1.4912504148810115, 1.091129247215256, 0.8818465230617194, 0.8715920116565027, 0.6503144256419822, 0.49863208841232076, 0.4624725447808252 ], "peak_layer": 21, "peak_normalized": 3.6056946953031805, "population_cv": 0.7234178171751035, "rest_mean": 1.2979781601003466e-05, "spike_contrast": 3.2968643466956427, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2792579188244416e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.2206347491883207e-05, 1.3508459232980385e-05, 1.12722400444909e-05, 1.0800607924466021e-05, 1.2900422007078305e-05, 1.0689462214941159e-05, 9.451743608224206e-06, 8.936035555962007e-06, 1.1220211490581278e-05, 1.0468403161212336e-05, 9.757372026797384e-06, 9.177146239380818e-06, 7.808387636032421e-06, 7.3296264417876955e-06, 7.410000762320124e-06, 9.259598300559446e-06, 2.2145992261357605e-05, 2.2417205400415696e-05, 2.0034221961395815e-05, 1.8763063053484075e-05, 6.359735562000424e-05, 4.9112488341052085e-05, 3.84151644539088e-05, 3.080127135035582e-05, 3.2036616175901145e-05, 2.6302721380488947e-05, 1.9245371731813066e-05, 1.5554036508547142e-05, 1.5373167116194963e-05, 1.1470266144897323e-05, 8.794888344709761e-06, 8.157105185091496e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.3983808154735016e-05, "first_to_last_ratio": 0.6452579224313538, "imbalance_abs_log_ratio": 0.43810516237842645, "last_quartile_mean": 6.816469294790295e-05, "mean": 7.033422900803998e-05, "normalized": [ 0.6809924600708206, 0.7475303658027369, 0.6323190441913445, 0.6077859654839745, 0.7056348319250383, 0.5942536325242065, 0.530225745030374, 0.5040918553425093, 0.6368908391102093, 0.5978963063844708, 0.5559498242106791, 0.5233025181432952, 0.44344812436490716, 0.41411157823288425, 0.41744328645242573, 0.5192458958427938, 1.2557243607052013, 1.2676253654309775, 1.1574430802993265, 1.060137110625566, 3.6189237818679025, 2.8123811167698722, 2.2018107363871295, 1.7616009064115667, 1.8179427908533603, 1.4871270732826793, 1.0868453970932423, 0.8727044844737096, 0.8717769668640218, 0.6523346900704022, 0.4999162674775265, 0.4645835982748448 ], "peak_layer": 21, "peak_normalized": 3.6189237818679025, "population_cv": 0.7299683580686596, "rest_mean": 5.154545740677147e-05, "spike_contrast": 3.3328561680843984, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017179359565488994, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.789707963936962e-05, 5.2576971938833594e-05, 4.4473672460298985e-05, 4.274815728422254e-05, 4.963028186466545e-05, 4.1796371078817174e-05, 3.729301897692494e-05, 3.545491199474782e-05, 4.479522613110021e-05, 4.205257573630661e-05, 3.910230225301348e-05, 3.680607915157452e-05, 3.1189581932267174e-05, 2.9126218578312546e-05, 2.9360551707213745e-05, 3.6520759749691933e-05, 8.832040475681424e-05, 8.915745274862275e-05, 8.140786667354405e-05, 7.456392631866038e-05, 0.0002545342140365392, 0.00019780665752477944, 0.00015486266056541353, 0.00012390084157232195, 0.00012786360457539558, 0.00010459593613632023, 7.644243305549026e-05, 6.138099706731737e-05, 6.13157608313486e-05, 4.588145748130046e-05, 3.516122524160892e-05, 3.267612919444218e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.039005281403660774, "first_to_last_ratio": 0.6452579411239904, "imbalance_abs_log_ratio": 0.4381051334091792, "last_quartile_mean": 0.06044913036748767, "mean": 0.06237309757852927, "normalized": [ 0.6809924890770891, 0.7475303831256814, 0.6323190938885936, 0.6077859577612714, 0.7056348041762752, 0.5942536183897815, 0.5302257656275956, 0.5040918555375298, 0.6368908140206065, 0.5978963019204037, 0.5559498364917699, 0.5233025167864885, 0.4434481328217215, 0.4141115698767486, 0.41744329043307676, 0.5192458729418684, 1.2557243339288315, 1.2676253196693095, 1.1574430718922224, 1.0601371340408086, 3.6189238450886454, 2.8123810331522527, 2.2018108921566237, 1.761600919246031, 1.8179427635119274, 1.487127014661369, 1.086845368525537, 0.8727044924490052, 0.8717770087213936, 0.6523346812884329, 0.4999162376710536, 0.46458358112005493 ], "peak_layer": 21, "peak_normalized": 3.6189238450886454, "population_cv": 0.7299683644727474, "rest_mean": 0.045711026768441555, "spike_contrast": 3.3328562214267015, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15234827995300293, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.042475610971450806, 0.046625785529613495, 0.03943970054388046, 0.03790949285030365, 0.04401262849569321, 0.03706543892621994, 0.03307182341814041, 0.03144177049398422, 0.03972485288977623, 0.03729264438152313, 0.034676313400268555, 0.0326399989426136, 0.027659233659505844, 0.02582942135632038, 0.02603723108768463, 0.03238697350025177, 0.07832341641187668, 0.0790657177567482, 0.07219330966472626, 0.06612403690814972, 0.2257234901189804, 0.17541691660881042, 0.13733376562595367, 0.10987650603055954, 0.1133907213807106, 0.0927567183971405, 0.06778991222381592, 0.05443328246474266, 0.054375432431697845, 0.04068813472986221, 0.031181324273347855, 0.028977517038583755 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8962810770517535e-06, "first_to_last_ratio": 0.40446687944391657, "imbalance_abs_log_ratio": 0.9051854260734966, "last_quartile_mean": 4.688346990633363e-06, "mean": 3.997706922831412e-06, "normalized": [ 0.617154002791989, 0.5425132643975126, 0.522616779881192, 0.5130533608023556, 0.4179497664431836, 0.4022508178223506, 0.3871440054888278, 0.39205556299135513, 0.47572519958981047, 0.4709737480952342, 0.46924028063152856, 0.46856760789403773, 0.43552024907631937, 0.4334760110167188, 0.4573112739567068, 0.48343767377708236, 0.9432261829651593, 0.9556187861127372, 0.9329616531367625, 0.91445072544449, 3.5932868350685228, 3.016174008278517, 2.572613821305946, 2.200605947741084, 2.1897185116963183, 1.8429631378409383, 1.3739246533430545, 1.1483111158298067, 1.0092131774108768, 0.7452469424242577, 0.5556201735322511, 0.5170747232130729 ], "peak_layer": 21, "peak_normalized": 3.5932868350685228, "population_cv": 0.8307963422495312, "rest_mean": 2.7284499108443724e-06, "spike_contrast": 3.9772380443675277, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0851694787561427e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.467200829414651e-06, 2.168809032809804e-06, 2.089268718918902e-06, 2.0510369722614996e-06, 1.670840674705687e-06, 1.6080808791230083e-06, 1.5476882708753692e-06, 1.5673232383051072e-06, 1.9018099237655406e-06, 1.8828150132321753e-06, 1.8758851183520164e-06, 1.8731959698925493e-06, 1.7410823147656629e-06, 1.7329100501228822e-06, 1.8281964457855793e-06, 1.932642135216156e-06, 3.7707418414356653e-06, 3.82028383683064e-06, 3.7297072594810743e-06, 3.6557059956976445e-06, 1.4364907656272408e-05, 1.2057779713359196e-05, 1.0284556083206553e-05, 8.797377631708514e-06, 8.753852853260469e-06, 7.367626494669821e-06, 5.492548098118277e-06, 4.590611297317082e-06, 4.034538505948149e-06, 2.9792788609483978e-06, 2.2212066141946707e-06, 2.067113200610038e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.395081762391783e-05, "first_to_last_ratio": 0.7815758779298154, "imbalance_abs_log_ratio": 0.246443041153289, "last_quartile_mean": 4.343892715041875e-05, "mean": 4.974787287892468e-05, "normalized": [ 0.7500101371289282, 0.8083785365909619, 0.6989343871323576, 0.6730384758360606, 0.7487834825409285, 0.6472056011592257, 0.5806038659736669, 0.5527069076510954, 0.6849789424977496, 0.6414689031465389, 0.5923280921669694, 0.5570919747617912, 0.5025384608304704, 0.4685796759451795, 0.4679593283316108, 0.5622406113180932, 1.3163192795863237, 1.3145042344644593, 1.1718309380215874, 1.0894177246512402, 3.678688405943946, 2.7746227673728385, 2.0966738705116796, 1.635642610362089, 1.6803690663352613, 1.3632931153614756, 0.986286743821586, 0.8095332329033809, 0.728391794299929, 0.5544865952259644, 0.4452050831707052, 0.41788715495590656 ], "peak_layer": 21, "peak_normalized": 3.678688405943946, "population_cv": 0.7109227971967927, "rest_mean": 3.7097179099671645e-05, "spike_contrast": 3.182495870364873, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011806161928689108, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.731140895979479e-05, 4.021511267637834e-05, 3.4770499041769654e-05, 3.3482232538517565e-05, 3.725038550328463e-05, 3.219710197299719e-05, 2.88838073174702e-05, 2.7495992981130257e-05, 3.4076245356118307e-05, 3.1911713449517265e-05, 2.9467062631738372e-05, 2.7714140742318705e-05, 2.5000219466164708e-05, 2.331084215256851e-05, 2.3279981178347953e-05, 2.79702744592214e-05, 6.548408418893814e-05, 6.539378955494612e-05, 5.8296096540289e-05, 5.419621447799727e-05, 0.0001830069231800735, 0.00013803158071823418, 0.00010430506517877802, 8.136974065564573e-05, 8.359478670172393e-05, 6.782093259971589e-05, 4.9065667553804815e-05, 4.027255636174232e-05, 3.623594238888472e-05, 2.7584528652369045e-05, 2.214800588262733e-05, 2.078899706248194e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6004547407865175e-05, "first_to_last_ratio": 1.0323157315408855, "imbalance_abs_log_ratio": 0.031804561684426046, "last_quartile_mean": 2.519049803595408e-05, "mean": 3.413751699099521e-05, "normalized": [ 0.838119937847341, 0.8931761838723624, 0.7929921852930517, 0.7615904360235652, 0.8189280439687685, 0.7189183912075784, 0.6527804223286999, 0.6175632597625974, 0.7513453279894857, 0.7058548167769552, 0.6474515253469071, 0.6094592164179198, 0.5931149886633756, 0.5496093129413688, 0.5439605510615998, 0.633537428927029, 1.4126123740339225, 1.4259512145086812, 1.2377137858897012, 1.131469831457994, 3.7563109168199906, 2.718723537642896, 1.8748185902822487, 1.4106982999267925, 1.4327179849324012, 1.1438362357528318, 0.8490271550305547, 0.7287425009809699, 0.5778680951931764, 0.464037753169555, 0.3681285307530028, 0.3389411651966756 ], "peak_layer": 21, "peak_normalized": 3.7563109168199906, "population_cv": 0.6884877712720907, "rest_mean": 2.6307041547692347e-05, "spike_contrast": 2.9050048917999454, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.642208438483067e-05, "top_five_layers": [ 21, 22, 23, 25, 18 ], "top_five_spike_overlap": 4 }, "values": [ 2.8611333618755452e-05, 3.0490817152895033e-05, 2.7070784199167974e-05, 2.599880644993391e-05, 2.7956170015386306e-05, 2.454208879498765e-05, 2.2284302758635022e-05, 2.1082076273160055e-05, 2.5649063900345936e-05, 2.4096130800899118e-05, 2.2102387447375804e-05, 2.0805424355785362e-05, 2.0247473003109917e-05, 1.8762297258945182e-05, 1.856946255429648e-05, 2.1627394744427875e-05, 4.822307892027311e-05, 4.867843381362036e-05, 4.225247539579868e-05, 3.862557059619576e-05, 0.00012823112774640322, 9.281047096010298e-05, 6.400165148079395e-05, 4.8157737182918936e-05, 4.890943455393426e-05, 3.9047728932928294e-05, 2.8983678930671886e-05, 2.4877459509298205e-05, 1.9726981918211095e-05, 1.5841096683288924e-05, 1.2566993973450735e-05, 1.1570609785849229e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.908438576327171e-05, "first_to_last_ratio": 0.6062325309661494, "imbalance_abs_log_ratio": 0.5004916520467982, "last_quartile_mean": 0.00014694755100208567, "mean": 0.00014725192045261792, "normalized": [ 0.6637009362863341, 0.7248732277041845, 0.6138698495621964, 0.598513900406796, 0.680481175716416, 0.565076258862256, 0.5041321503581428, 0.4891880965619703, 0.6294201840754665, 0.5850404355839129, 0.5479525097707834, 0.5179728424254814, 0.4076252060436077, 0.3853916272098706, 0.3845475777225467, 0.4800015848780767, 1.1784027781257396, 1.1715870365157128, 1.1011549065529929, 1.0246566094283616, 3.66881165389385, 2.9055737038215397, 2.326629675317946, 1.8619320585590522, 1.8930003135554039, 1.5415479300660813, 1.1509068852552828, 0.9338288403787282, 0.8717742548708616, 0.6455051030286945, 0.49036587220767053, 0.45653481525404077 ], "peak_layer": 21, "peak_normalized": 3.66881165389385, "population_cv": 0.7616800575204435, "rest_mean": 0.00010549810718253462, "spike_contrast": 3.5329781933070796, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00037272251211106777, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 9.77312374743633e-05, 0.00010673897486412898, 9.039351425599307e-05, 8.81323212524876e-05, 0.0001002021599560976, 8.320856431964785e-05, 7.423442730214447e-05, 7.203388668131083e-05, 9.268333087675273e-05, 8.614832768216729e-05, 8.068705938057974e-05, 7.627249578945339e-05, 6.002359441481531e-05, 5.674965723301284e-05, 5.662536932504736e-05, 7.068115519359708e-05, 0.00017352207214571536, 0.00017251844110433012, 0.00016214717470575124, 0.0001508826535427943, 0.0005402395618148148, 0.0004278513079043478, 0.00034260068787261844, 0.0002741730713751167, 0.00027874793158844113, 0.00022699589317198843, 0.00016947324911598116, 0.0001375080901198089, 0.00012837043323088437, 9.505186608294025e-05, 7.220731640700251e-05, 6.722562829963863e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.09253507107496262, 0.30484533309936523, 0.33420246839523315, 0.38682517409324646, 0.12552575767040253, 0.14749252796173096, 0.1726136952638626, 0.1892337054014206, 0.031880609691143036, 0.06123264133930206, 0.10403046011924744, 0.15778414905071259, 0.053015924990177155, 0.09510950744152069, 0.1597633957862854, 0.7120755910873413, 0.04075472429394722, 0.07218439877033234, 0.11449843645095825, 0.16539782285690308, 0.0493597537279129, 0.07179203629493713, 0.10295005142688751, 0.1323672980070114, 0.023035654798150063, 0.06294231116771698, 0.09175154566764832, 0.1252463310956955, 0.03424545377492905, 0.07851024717092514, 0.12469635903835297, 0.19299884140491486 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1261542454121809e-05, "first_to_last_ratio": 0.5245675018823146, "imbalance_abs_log_ratio": 0.6451811618435493, "last_quartile_mean": 2.146824272131198e-05, "mean": 2.008613361681455e-05, "normalized": [ 0.6085515532483275, 0.6723602464807182, 0.5609760959264884, 0.53758893439475, 0.6415342139885134, 0.5340779441503937, 0.4789823840329274, 0.45122885288223713, 0.5900862077803115, 0.5406841208808664, 0.5086899166409938, 0.47350367257308246, 0.40488038567700324, 0.38162938029260524, 0.36548145139289145, 0.4467012874253964, 1.1143303455046147, 1.1164970711323288, 1.0360116270365431, 0.9253438018068647, 3.771407372914525, 3.0487664316584073, 2.367876661366877, 1.872337110236809, 1.790253835037814, 1.5773117786405777, 1.2887586410545064, 0.9509019452609131, 1.1740685268722015, 0.7631964556770826, 0.5687161676492125, 0.4372655803832159 ], "peak_layer": 21, "peak_normalized": 3.771407372914525, "population_cv": 0.7926334965261557, "rest_mean": 1.42457990848386e-05, "spike_contrast": 3.6238009382307355, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.1623940089484675e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.2223447811265942e-05, 1.3505117749446072e-05, 1.1267840818618424e-05, 1.07980831671739e-05, 1.2885941941931378e-05, 1.0727560947998427e-05, 9.62090416578576e-06, 9.06344303075457e-06, 1.1852550414914731e-05, 1.0860253496502992e-05, 1.0217613635177258e-05, 9.51085803535534e-06, 8.132481525535695e-06, 7.665458724659402e-06, 7.34110926714493e-06, 8.972501746029593e-06, 2.2382588213076815e-05, 2.2426109353546053e-05, 2.0809467969229445e-05, 1.8586579244583845e-05, 7.575299241580069e-05, 6.123792991274968e-05, 4.756148700835183e-05, 3.760801337193698e-05, 3.5959277738584206e-05, 3.168209514115006e-05, 2.5886178264045157e-05, 1.9099943528999574e-05, 2.3582497306051664e-05, 1.5329665984609164e-05, 1.1423308933444787e-05, 8.782974873611238e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.4184340367792174e-05, "first_to_last_ratio": 0.5155127283755759, "imbalance_abs_log_ratio": 0.6625932845097232, "last_quartile_mean": 8.570950421926682e-05, "mean": 8.019429117211985e-05, "normalized": [ 0.5977553912963086, 0.6560764337279869, 0.5549115575710053, 0.5332632715493767, 0.618731238431461, 0.5235220032899371, 0.4739610390296197, 0.44950831585208795, 0.5901295178770004, 0.5434863568103182, 0.5105118554999238, 0.4751981306058326, 0.4052536442822589, 0.3803760366271764, 0.36329074496540736, 0.44135367311072854, 1.112029465926781, 1.1101484697729092, 1.04807895367264, 0.9240971550402413, 3.7925041485004933, 3.0713698535863756, 2.383447135178006, 1.890810504439103, 1.7925006015588865, 1.5758247605004587, 1.2855904653729862, 0.9466104071448787, 1.1794713255182339, 0.7622315023149855, 0.5700532953493974, 0.43790274559719394 ], "peak_layer": 21, "peak_normalized": 3.7925041485004933, "population_cv": 0.7995548542275265, "rest_mean": 5.6639052976000435e-05, "spike_contrast": 3.6616533387138865, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002073925774311647, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 4.79365698993206e-05, 5.2613584557548165e-05, 4.450073902262375e-05, 4.276467007002793e-05, 4.96187130920589e-05, 4.198347596684471e-05, 3.800896956818178e-05, 3.604800076573156e-05, 4.732501838589087e-05, 4.358450314612128e-05, 4.094013638678007e-05, 3.8108177250251174e-05, 3.249902874813415e-05, 3.0503986636176705e-05, 2.913384378189221e-05, 3.5394044971326366e-05, 8.917841478250921e-05, 8.902756962925196e-05, 8.40499487821944e-05, 7.410731632262468e-05, 0.000304137181956321, 0.0002463063283357769, 0.00019113885355181992, 0.00015163220814429224, 0.00014374831516761333, 0.0001263721496798098, 0.0001030970161082223, 7.591275061713532e-05, 9.45868669077754e-05, 6.112661503721029e-05, 4.571501995087601e-05, 3.511730028549209e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03918311558663845, "first_to_last_ratio": 0.5155127429948628, "imbalance_abs_log_ratio": 0.6625932561509924, "last_quartile_mean": 0.07600804464891553, "mean": 0.07111709873424843, "normalized": [ 0.5977553845258899, 0.656076418540609, 0.5549115935781082, 0.5332632748164047, 0.6187312693383394, 0.5235220185307409, 0.4739610644907502, 0.4495083446741889, 0.5901295424808332, 0.5434863451914381, 0.5105118857031089, 0.4751981292368421, 0.4052536577066065, 0.38037603736972314, 0.36329075680382195, 0.4413536490689804, 1.1120294326243692, 1.1101484823828944, 1.0480789054186883, 0.9240971084727143, 3.792504006439516, 3.0713698521253847, 2.383447240422923, 1.8908105107644724, 1.7925005311890823, 1.575824785560249, 1.2855905174797924, 0.9466103668129862, 1.1794713571647604, 0.7622314608572955, 0.5700533268339548, 0.4379027433945313 ], "peak_layer": 21, "peak_normalized": 3.792504006439516, "population_cv": 0.7995548405481961, "rest_mean": 0.05022807840119909, "spike_contrast": 3.6616532900913086, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.18391780853271483, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.04251062870025635, 0.04665825143456459, 0.03946370258927345, 0.037924136966466904, 0.04400237277150154, 0.03723136708140373, 0.03370673581957817, 0.031967729330062866, 0.04196830093860626, 0.03865117207169533, 0.036306124180555344, 0.03379471227526665, 0.02882046438753605, 0.02705124020576477, 0.02583618462085724, 0.03138779103755951, 0.07908430695533752, 0.07895053923130035, 0.07453633099794388, 0.0657191053032875, 0.2697118818759918, 0.218426913022995, 0.169503852725029, 0.1344689577817917, 0.12747743725776672, 0.11206808686256409, 0.09142746776342392, 0.06732018291950226, 0.08388058096170425, 0.0542076900601387, 0.04054053872823715, 0.031142372637987137 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8926682940900719e-06, "first_to_last_ratio": 0.32617819149698246, "imbalance_abs_log_ratio": 1.1203114472626277, "last_quartile_mean": 5.802559286394171e-06, "mean": 4.670736920786567e-06, "normalized": [ 0.5310214304016295, 0.4614624888755425, 0.44477058805262604, 0.4368101139545488, 0.3555922725307191, 0.3424096008550773, 0.333381953461308, 0.3362983274867149, 0.4224036639064773, 0.41210428787587466, 0.41438837559751834, 0.41562174402282875, 0.3825706733995329, 0.3815673445421813, 0.3895113889811354, 0.40964670295389133, 0.8179200324465078, 0.8423754461799106, 0.8552890122092341, 0.8530595439765862, 3.8961508812147314, 3.3402234019552477, 2.714242582301369, 2.272602051203693, 2.0707052479706927, 1.8581861711397907, 1.5332291278146746, 1.1680063245911945, 1.3343957866516452, 0.8510283999847308, 0.6469666788242595, 0.4760583546381262 ], "peak_layer": 21, "peak_normalized": 3.8961508812147314, "population_cv": 0.9180173062016742, "rest_mean": 3.0629785972754084e-06, "spike_contrast": 4.359361792349546, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3352631867746823e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.4802614007057855e-06, 2.1553698843490565e-06, 2.077406406897353e-06, 2.0402251266204985e-06, 1.6608779560556286e-06, 1.5993051647456014e-06, 1.5571393987556803e-06, 1.5707610145909712e-06, 1.972936388483504e-06, 1.9248307125963038e-06, 1.9354990854481002e-06, 1.94125982488913e-06, 1.7868869690573774e-06, 1.7822006839196547e-06, 1.8193052255810471e-06, 1.9133519799652277e-06, 3.820289293798851e-06, 3.9345140976365656e-06, 3.994829967268743e-06, 3.9844167076807935e-06, 1.8197895769844763e-05, 1.5601304767187685e-05, 1.2677513041126076e-05, 1.0614726306812372e-05, 9.671719453763217e-06, 8.679098755237646e-06, 7.161309895309387e-06, 5.45545026398031e-06, 6.232611667655874e-06, 3.9749297684466e-06, 3.0218111533031333e-06, 2.2235433334572008e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.407248482290015e-05, "first_to_last_ratio": 0.6342850678385603, "imbalance_abs_log_ratio": 0.4552567917851402, "last_quartile_mean": 5.371793622543919e-05, "mean": 5.588281288737562e-05, "normalized": [ 0.667605556447693, 0.7197799315075553, 0.6218050056576038, 0.5988650842787754, 0.6666530112346837, 0.5782950032081196, 0.5252885832899008, 0.4994128735568847, 0.6402020441359789, 0.5894915107696465, 0.5499104969232754, 0.5108819879445103, 0.46466609827668737, 0.43567603051211307, 0.41544997111847654, 0.48809065762969883, 1.1821280091855806, 1.1664813213195084, 1.07816907868966, 0.9625539685607898, 3.8682944428485135, 3.0459807530568472, 2.2760455923564815, 1.7581896219738866, 1.7091546359484804, 1.4520780986698394, 1.1543285578371303, 0.8578398764689898, 0.9984751177727861, 0.6390807593663731, 0.4862828593592872, 0.39284346009424276 ], "peak_layer": 21, "peak_normalized": 3.8682944428485135, "population_cv": 0.7787255378412773, "rest_mean": 4.0033484630704925e-05, "spike_contrast": 3.5337714610257334, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014146918547339738, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.7307676393538713e-05, 4.022332723252475e-05, 3.474821278359741e-05, 3.346626544953324e-05, 3.725444548763335e-05, 3.231675145798363e-05, 2.935460361186415e-05, 2.7908596166525967e-05, 3.57762910425663e-05, 3.2942443795036525e-05, 3.073054540436715e-05, 2.854952253983356e-05, 2.5966848625103012e-05, 2.4346802092622966e-05, 2.321651300007943e-05, 2.7275878892396577e-05, 6.606063834624365e-05, 6.518625741591677e-05, 6.0251120885368437e-05, 5.379022331908345e-05, 0.0002161711745429784, 0.0001702179724816233, 0.00012719182996079326, 9.825258166529238e-05, 9.551236871629953e-05, 8.114620868582278e-05, 6.45071268081665e-05, 4.7938505304045975e-05, 5.579759817919694e-05, 3.571363049559295e-05, 2.7174854039913043e-05, 2.195319757447578e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.60414058175229e-05, "first_to_last_ratio": 0.8644500280195812, "imbalance_abs_log_ratio": 0.14566178001816363, "last_quartile_mean": 3.0124825002530997e-05, "mean": 3.7509687871306596e-05, "normalized": [ 0.7612877820794312, 0.810711701779243, 0.7209397300058344, 0.6925594766630951, 0.7442562064075515, 0.6550198977679047, 0.601099191291831, 0.5681910666111114, 0.7147935701981128, 0.6606027044500686, 0.6100202180357266, 0.5720600519271453, 0.5632479423839041, 0.5253282201512559, 0.49767906018209906, 0.5644858935407635, 1.2937489635148631, 1.2840324435565709, 1.1578011905188617, 1.0245733409201465, 3.982499346289561, 3.0010314180971562, 2.0616055293573603, 1.5074555677980233, 1.505937419140994, 1.2140551303841993, 0.9515353865179874, 0.7371695567558888, 0.7704610253685644, 0.5109788396566676, 0.4050543403397244, 0.3297777883083533 ], "peak_layer": 21, "peak_normalized": 3.982499346289561, "population_cv": 0.7526960469489694, "rest_mean": 2.7703642310236183e-05, "spike_contrast": 3.2653588610499065, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.046233390108682e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.855566708603874e-05, 3.04095428873552e-05, 2.70422242465429e-05, 2.5977689801948145e-05, 2.7916817998629995e-05, 2.4569591914769262e-05, 2.2547043045051396e-05, 2.1312669559847564e-05, 2.6811683710548095e-05, 2.4779001250863075e-05, 2.28816679737065e-05, 2.1457793991430663e-05, 2.1127254512975924e-05, 1.9704897567862645e-05, 1.8667786207515746e-05, 2.1173689674469642e-05, 4.852811980526894e-05, 4.8163656174438074e-05, 4.342876127338968e-05, 3.84314262191765e-05, 0.00014938230742700398, 0.00011256775178480893, 7.733017991995439e-05, 5.654418782796711e-05, 5.64872425456997e-05, 4.5538828999269754e-05, 3.569179534679279e-05, 2.765099998214282e-05, 2.8899752578581683e-05, 1.9166656784364022e-05, 1.5193461877061054e-05, 1.2369861906336155e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.932642231229693e-05, "first_to_last_ratio": 0.48429290534855984, "imbalance_abs_log_ratio": 0.7250653789708609, "last_quartile_mean": 0.00018444710076437332, "mean": 0.00016840073044477322, "normalized": [ 0.5799699418435614, 0.631638262959337, 0.5372322150726369, 0.519448365501922, 0.5960027732717111, 0.4981229459623488, 0.4492943531121762, 0.43180823754645586, 0.5789679033812432, 0.5298804189104165, 0.5013705840728442, 0.4662175781515477, 0.3655512526294271, 0.35395025516272205, 0.3328677409454649, 0.4094748170873465, 1.0315739694159025, 1.0322187789196813, 0.9834616923499129, 0.8864297869456468, 3.853800867660393, 3.163217666231461, 2.5024524541124884, 2.002752567834613, 1.843634113893829, 1.6377082968935632, 1.3336197815157738, 1.0052459186517901, 1.1908891844448808, 0.7572668908929535, 0.5664028889185376, 0.4275274957074122 ], "peak_layer": 21, "peak_normalized": 3.853800867660393, "population_cv": 0.8345380635674979, "rest_mean": 0.00011622233998625436, "spike_contrast": 3.873300425494927, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00045016403892077505, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.766736184246838e-05, 0.00010636834485922009, 9.047029743669555e-05, 8.747548417886719e-05, 0.00010036730236606672, 8.388426795136184e-05, 7.566149724880233e-05, 7.271682261489332e-05, 9.749861783348024e-05, 8.923224959289655e-05, 8.443117258138955e-05, 7.851138070691377e-05, 6.155909795779735e-05, 5.9605481510516256e-05, 5.605517071671784e-05, 6.895585829624906e-05, 0.00017371780995745212, 0.0001738263963488862, 0.00016561566735617816, 0.00014927542360965163, 0.000648982881102711, 0.0005326881655491889, 0.0004214148211758584, 0.0003372649953234941, 0.000310469331452623, 0.00027579127345234156, 0.00022458254534285516, 0.00016928414697758853, 0.00020054660853929818, 0.00012752429756801575, 9.538266021991149e-05, 7.199594256235287e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.15345558524131775, 0.3249167501926422, 0.3818626403808594, 0.40600064396858215, 0.1351035088300705, 0.15324510633945465, 0.18001793324947357, 0.20391276478767395, 0.04374876990914345, 0.08211024105548859, 0.1380448192358017, 0.2295975536108017, 0.08199761807918549, 0.13892096281051636, 0.21687114238739014, 0.7360114455223083, 0.057135771960020065, 0.08251863718032837, 0.12661415338516235, 0.19619159400463104, 0.06753352284431458, 0.095451220870018, 0.1274663656949997, 0.15628865361213684, 0.058436937630176544, 0.08794140070676804, 0.12247690558433533, 0.1640285700559616, 0.07537907361984253, 0.12290703505277634, 0.19157102704048157, 0.29249706864356995 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1220713531656656e-05, "first_to_last_ratio": 0.6555413366591524, "imbalance_abs_log_ratio": 0.4222939165017446, "last_quartile_mean": 1.7116713934228756e-05, "mean": 1.7638015336274293e-05, "normalized": [ 0.6920608231884311, 0.7659177824262653, 0.6390754033749174, 0.6123424804403663, 0.7313911494807722, 0.6060429021656846, 0.5358717188243214, 0.5066293084101811, 0.6361394778273108, 0.5935071698997143, 0.5532047504443424, 0.5203003869133469, 0.4427001574083234, 0.41555518288954896, 0.42010731902783316, 0.5249828006548773, 1.2555950689969066, 1.2709301325611273, 1.1358461030518159, 1.0637808540376092, 3.605678554956403, 2.784492491030135, 2.1780228929741425, 1.7462695606216458, 1.8163019324419567, 1.491254439501435, 1.0911345614825427, 0.8818559802285882, 0.8715835151550508, 0.6503123816361532, 0.4986363533333601, 0.46247636461489006 ], "peak_layer": 21, "peak_normalized": 3.605678554956403, "population_cv": 0.7234183193184713, "rest_mean": 1.2979772741851371e-05, "spike_contrast": 3.2968624487684486, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2792525346158075e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.220657941303216e-05, 1.3509269592759665e-05, 1.1272021765762474e-05, 1.0800506061059423e-05, 1.2900288311357144e-05, 1.0689394002838526e-05, 9.451713594899047e-06, 8.935935511544812e-06, 1.122023786592763e-05, 1.0468288564879913e-05, 9.757433872437105e-06, 9.177066203847062e-06, 7.808352165739052e-06, 7.329568688874133e-06, 7.409859335893998e-06, 9.259654689230956e-06, 2.214620508311782e-05, 2.2416685169446282e-05, 2.003407098527532e-05, 1.876298301795032e-05, 6.359701364999637e-05, 4.911292126053013e-05, 3.841600118903443e-05, 3.080072929151356e-05, 3.203596133971587e-05, 2.630276867421344e-05, 1.9245448129368015e-05, 1.555418930365704e-05, 1.5373003407148644e-05, 1.1470219760667533e-05, 8.794955647317693e-06, 8.157165211741813e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.398412329464918e-05, "first_to_last_ratio": 0.6452634668607965, "imbalance_abs_log_ratio": 0.43809656983670076, "last_quartile_mean": 6.816459563196986e-05, "mean": 7.03343731061068e-05, "normalized": [ 0.6809932372926891, 0.7475632825133391, 0.6323153694166137, 0.6077851857944574, 0.7056447138179579, 0.5942446047107566, 0.530223262177042, 0.504089839823597, 0.6368974480554273, 0.5978888745536934, 0.5559539610577222, 0.5232972563751404, 0.4434509399790644, 0.41411238498840724, 0.4174418363858031, 0.5192489699581264, 1.2557404086895352, 1.2676084925356499, 1.1574347089891701, 1.060138766241709, 3.6189242296270336, 2.81239769967395, 2.2018329150221976, 1.7615672973682381, 1.8179181698124194, 1.4871187506667403, 1.086854135909831, 0.872703317202576, 0.8717669049561169, 0.6523365087620864, 0.4999226915339381, 0.46458383610897114 ], "peak_layer": 21, "peak_normalized": 3.6189242296270336, "population_cv": 0.7299683279122143, "rest_mean": 5.15456125598306e-05, "spike_contrast": 3.3328477735441013, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017179368005599828, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.789723243447952e-05, 5.257939483271912e-05, 4.447350511327386e-05, 4.274819002603181e-05, 4.96310785820242e-05, 4.17958217440173e-05, 3.729292075149715e-05, 3.545484287315048e-05, 4.479578274185769e-05, 4.205213917884976e-05, 3.91026733268518e-05, 3.680578447529115e-05, 3.1189843866741285e-05, 2.9126334993634373e-05, 2.9360509870457463e-05, 3.65210507879965e-05, 8.83217144291848e-05, 8.915644866647199e-05, 8.140744466800243e-05, 7.45641955290921e-05, 0.0002545347670093179, 0.0001978082291316241, 0.000154864537762478, 0.00012389873154461384, 0.00012786213483195752, 0.00010459556506248191, 7.644320430699736e-05, 6.138104072306305e-05, 6.131517875473946e-05, 4.588167939800769e-05, 3.516174911055714e-05, 3.2676212867954746e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.039005559869110584, "first_to_last_ratio": 0.6452634722813873, "imbalance_abs_log_ratio": 0.43809656143611553, "last_quartile_mean": 0.06044904375448823, "mean": 0.06237322464585304, "normalized": [ 0.6809932518812878, 0.7475633220322179, 0.6323153569646397, 0.6077851973803072, 0.7056447145510248, 0.5942445836918125, 0.530223252027616, 0.5040898132583496, 0.6368974600710677, 0.5978889321237267, 0.5559539597752757, 0.523297269904143, 0.4434509324241475, 0.4141123687040429, 0.4174418427557242, 0.5192489959347639, 1.2557404102027183, 1.2676085225122937, 1.157434741358947, 1.0601387967701486, 3.61892435638739, 2.812397521737949, 2.201832924858681, 1.7615672286963322, 1.8179181559541986, 1.487118729202359, 1.086854143939758, 0.8727033118250799, 0.8717668711180101, 0.6523365178130685, 0.4999226849610752, 0.46458382918184327 ], "peak_layer": 21, "peak_normalized": 3.61892435638739, "population_cv": 0.7299683256836244, "rest_mean": 0.045711164397222025, "spike_contrast": 3.3328477188762906, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15234834998846053, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.04247574508190155, 0.04662793502211571, 0.039439547806978226, 0.03790952265262604, 0.044013336300849915, 0.03706495091319084, 0.03307173401117325, 0.03144170716404915, 0.039725348353385925, 0.03729226067662239, 0.03467664122581482, 0.03263973817229271, 0.02765946462750435, 0.02582952380180359, 0.026037193834781647, 0.03238723427057266, 0.0783245787024498, 0.07906483113765717, 0.07219293713569641, 0.06612427532672882, 0.2257239818572998, 0.17541830241680145, 0.1373354196548462, 0.10987462848424911, 0.11338941752910614, 0.09275639057159424, 0.06779059767723083, 0.054433319717645645, 0.05437491089105606, 0.04068833217024803, 0.031181789934635162, 0.028977591544389725 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.896312141980161e-06, "first_to_last_ratio": 0.40447227453095713, "imbalance_abs_log_ratio": 0.905172087401449, "last_quartile_mean": 4.688361258331497e-06, "mean": 3.997668517996544e-06, "normalized": [ 0.6172058310645722, 0.5425329228673286, 0.5226213455478163, 0.5130242205418598, 0.4179738021592733, 0.4022397236307717, 0.38715722309433015, 0.3920811131204843, 0.47573503087335345, 0.47091681751525505, 0.4692356598433531, 0.46858709630953144, 0.435518404119361, 0.43348810961922396, 0.457344361488258, 0.4834240606909693, 0.943243320831244, 0.9556851843891272, 0.9328847891755226, 0.9144352241016374, 3.593133434897055, 3.0161686306352355, 2.572714523011844, 2.200458051372808, 2.189693136620607, 1.8429744726585007, 1.3739587829516955, 1.1483183935808434, 1.0092761091983693, 0.7452413615169265, 0.5556410385724699, 0.5170878240003723 ], "peak_layer": 21, "peak_normalized": 3.593133434897055, "population_cv": 0.8307777929041875, "rest_mean": 2.728457953135485e-06, "spike_contrast": 3.9771203202072667, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0851405568246264e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.4673843199707335e-06, 2.1688667857233668e-06, 2.0892668999294983e-06, 2.050900775429909e-06, 1.670920710239443e-06, 1.6080210798463668e-06, 1.5477262422791682e-06, 1.567410322422802e-06, 1.901830955830519e-06, 1.8825693359758588e-06, 1.8758486248771078e-06, 1.8732558828560286e-06, 1.741058213156066e-06, 1.7329417687506066e-06, 1.8283111558048404e-06, 1.9325691482663387e-06, 3.770774128497578e-06, 3.820512574748136e-06, 3.7293641526048305e-06, 3.6556089071382303e-06, 1.4364156413648743e-05, 1.2057642379659228e-05, 1.0284859854436945e-05, 8.796701877145097e-06, 8.753667316341307e-06, 7.3676010288181715e-06, 5.49263177163084e-06, 4.590596290654503e-06, 4.0347513277083635e-06, 2.9792279292450985e-06, 2.2212686872080667e-06, 2.0671457150456263e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.3950954957617796e-05, "first_to_last_ratio": 0.7815786467251918, "imbalance_abs_log_ratio": 0.24643949857917652, "last_quartile_mean": 4.343894897829159e-05, "mean": 4.974799179535694e-05, "normalized": [ 0.7500094412446059, 0.8083859646655934, 0.6989360803110847, 0.6730394265060721, 0.7487882742003255, 0.6472039809659893, 0.5806034653418889, 0.5527037948336986, 0.6849833016487895, 0.641463932771085, 0.5923270784846439, 0.5570934219720602, 0.5025383930607131, 0.4685801281186291, 0.467958282860767, 0.5622422290415673, 1.3163199357516149, 1.3145043099431595, 1.1718295263393368, 1.089421848322846, 3.6786851702332073, 2.7746307606104894, 2.0966820217393844, 1.6356296326692725, 1.680361539469081, 1.3632870777077486, 0.9862895783238333, 0.8095320290956769, 0.7283886637343489, 0.5544846482049088, 0.4452099057793211, 0.417886156048257 ], "peak_layer": 21, "peak_normalized": 3.6786851702332073, "population_cv": 0.710922698103497, "rest_mean": 3.709728177151798e-05, "spike_contrast": 3.18249263251226, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011806182592408732, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.7311463529476896e-05, 4.021557833766565e-05, 3.477066638879478e-05, 3.348235986777581e-05, 3.725071292137727e-05, 3.219709833501838e-05, 2.8883856430184096e-05, 2.7495903850649484e-05, 3.407654367038049e-05, 3.191154246451333e-05, 2.946708264062181e-05, 2.7714278985513374e-05, 2.5000275854836218e-05, 2.3310920369112864e-05, 2.327998481632676e-05, 2.7970421797363088e-05, 6.548427336383611e-05, 6.539394962601364e-05, 5.8296165661886334e-05, 5.4196549172047526e-05, 0.00018300719966646284, 0.0001380323083139956, 0.0001043057200149633, 8.136928954627365e-05, 8.359461207874119e-05, 6.782079435652122e-05, 4.906592585030012e-05, 4.027259274153039e-05, 3.623587326728739e-05, 2.7584497729549184e-05, 2.21482987399213e-05, 2.078899706248194e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.600453626655508e-05, "first_to_last_ratio": 1.032180910721196, "imbalance_abs_log_ratio": 0.031673952782361334, "last_quartile_mean": 2.5193777560161834e-05, "mean": 3.414015603198095e-05, "normalized": [ 0.8380570158080917, 0.8932999084125006, 0.7929620023886231, 0.7615443521416788, 0.8187638282400099, 0.7186815599217726, 0.6527123798513835, 0.6175741300582618, 0.7513411682592698, 0.705872928051971, 0.6475562024934578, 0.6094685286755085, 0.5930459106503145, 0.5494388493370179, 0.5439464215496852, 0.6333820560847124, 1.4124654565316987, 1.425994860990475, 1.2380543674563398, 1.1313103339756791, 3.756835524951107, 2.7186060870583466, 1.87468091251249, 1.410793634954463, 1.4329939421834126, 1.1438971076949418, 0.8490265267268121, 0.7287288463621843, 0.5779352074283672, 0.4639524923457935, 0.36814597511060676, 0.33893148179302335 ], "peak_layer": 21, "peak_normalized": 3.756835524951107, "population_cv": 0.6885594503102112, "rest_mean": 2.6308265020176163e-05, "spike_contrast": 2.9052606637917697, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.643236749572679e-05, "top_five_layers": [ 21, 22, 23, 25, 18 ], "top_five_spike_overlap": 4 }, "values": [ 2.8611397283384576e-05, 3.0497398256557062e-05, 2.7071846488979645e-05, 2.599924300739076e-05, 2.795272484945599e-05, 2.4535900593036786e-05, 2.228370249213185e-05, 2.1084077161503956e-05, 2.5650904717622325e-05, 2.4098611902445555e-05, 2.21076697926037e-05, 2.0807350665563717e-05, 2.024667992372997e-05, 1.8757928046397865e-05, 1.8570415704743937e-05, 2.1623762222588994e-05, 4.82217910757754e-05, 4.86836870550178e-05, 4.226736928103492e-05, 3.8623111322522163e-05, 0.00012825895100831985, 9.281363600166515e-05, 6.400189886335284e-05, 4.816471482627094e-05, 4.89226367790252e-05, 3.905282574123703e-05, 2.898589809774421e-05, 2.4878916519810446e-05, 1.9730798157979734e-05, 1.5839410480111837e-05, 1.256856103282189e-05, 1.1571173672564328e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.909050666261464e-05, "first_to_last_ratio": 0.6062699294291527, "imbalance_abs_log_ratio": 0.5004299639855584, "last_quartile_mean": 0.00014694858236907749, "mean": 0.00014725223013556388, "normalized": [ 0.6636255714147841, 0.7248917149077715, 0.6138660879695628, 0.5985975801010156, 0.6806345015153643, 0.5652332367621573, 0.5042378190281714, 0.4890714447869022, 0.6294477660992072, 0.5850399957826026, 0.5478904329704705, 0.518038310415621, 0.40778029155663986, 0.3853440981019431, 0.3845833582250337, 0.48007518680371525, 1.178544482689773, 1.1716070054101007, 1.1012637666749874, 1.0246953672401915, 3.668461219741745, 2.9054304267727518, 2.326469432391579, 1.8616676451976302, 1.8927854440285927, 1.5414183922061209, 1.1508159193509495, 0.9341585266360799, 0.8717798331906053, 0.6455697098701952, 0.49047863567768574, 0.45649679648005004 ], "peak_layer": 21, "peak_normalized": 3.668461219741745, "population_cv": 0.7615887971484168, "rest_mean": 0.00010550450948627528, "spike_contrast": 3.5324549012779802, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003726899216417223, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 9.772034536581486e-05, 0.00010674192162696272, 9.039315045811236e-05, 8.814482862362638e-05, 0.00010022494825534523, 8.323185465997085e-05, 7.42501433705911e-05, 7.201686094049364e-05, 9.268758731195703e-05, 8.614844409748912e-05, 8.067808812484145e-05, 7.628229650435969e-05, 6.004655733704567e-05, 5.674277781508863e-05, 5.663075717166066e-05, 7.06921418895945e-05, 0.00017354330339003354, 0.00017252174438908696, 0.00016216354561038315, 0.00015088867803569883, 0.0005401890957728028, 0.0004278311098460108, 0.00034257781226187944, 0.00027413471252657473, 0.0002787168778013438, 0.00022697729582432657, 0.00016946021059993654, 0.0001375569263473153, 0.0001283715246245265, 9.506157948635519e-05, 7.222407293738797e-05, 6.722017133142799e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.06279198825359344, 0.07690861076116562, 0.18058517575263977, 0.15882407128810883, 0.19892723858356476, 0.15647411346435547, 0.1533493995666504, 0.1527898609638214, 0.16355258226394653, 0.11789151281118393, 0.11456070095300674, 0.12148918211460114, 0.15094918012619019, 0.09854138642549515, 0.11213528364896774, 0.16254977881908417, 0.2323070764541626, 0.1390199512243271, 0.14812250435352325, 0.15333011746406555, 0.1781008094549179, 0.07593096792697906, 0.09691466391086578, 0.09368696808815002, 0.10724716633558273, 0.06538715958595276, 0.07412076741456985, 0.07655765861272812, 0.08420819044113159, 0.07230556756258011, 0.07723049819469452, 0.08422473073005676 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.792817086738069e-06, "first_to_last_ratio": 0.9644943347481165, "imbalance_abs_log_ratio": 0.03615132042434886, "last_quartile_mean": 8.079691923512655e-06, "mean": 1.231628780118399e-05, "normalized": [ 0.899719870687867, 0.727426048108731, 0.7370807482678542, 0.3965794568165141, 0.5003083522454715, 0.7486231449014935, 0.6212780315806493, 0.4307803771911971, 0.40860273125956736, 0.5996948368722662, 0.5787702645812915, 0.5364690416901848, 0.2710899543731052, 0.3344315013927567, 0.25145386433912675, 1.0500832430459097, 0.7256359007107253, 1.3787305863467312, 2.2756402165685268, 2.51490055711116, 3.4405522362788585, 3.2818395195720442, 2.448485364965397, 1.5936896131932947, 0.9036087632058596, 1.0199413906977872, 1.0227020812544427, 0.5747747031450406, 0.6830971927098096, 0.5175337025733492, 0.3003192317874535, 0.22615747252553492 ], "peak_layer": 21, "peak_normalized": 3.4405522362788585, "population_cv": 0.8497149496404188, "rest_mean": 9.274540818498483e-06, "spike_contrast": 3.098991321528187, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8741721507685725e-05, "top_five_layers": [ 21, 22, 20, 23, 19 ], "top_five_spike_overlap": 3 }, "values": [ 1.1081208867835812e-05, 8.959188562585041e-06, 9.078098628378939e-06, 4.884386726189405e-06, 6.161941655591363e-06, 9.220258107234258e-06, 7.651839041500352e-06, 5.3056151045893785e-06, 5.032468834542669e-06, 7.386014203802915e-06, 7.128301149350591e-06, 6.6073071138816886e-06, 3.338821898069e-06, 4.118954620935256e-06, 3.0969781619205605e-06, 1.293312743655406e-05, 8.937140592024662e-06, 1.6980842701741494e-05, 2.8027439839206636e-05, 3.0974239052738994e-05, 4.23748315370176e-05, 4.0420080040348694e-05, 3.0156250431900844e-05, 1.9628339941846207e-05, 1.112910558731528e-05, 1.2561891708173789e-05, 1.259589316759957e-05, 7.079090664774412e-06, 8.413221621594857e-06, 6.374094027705723e-06, 3.6988180909247603e-06, 2.785420520012849e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.7120979211758822e-05, "first_to_last_ratio": 0.8594772224120714, "imbalance_abs_log_ratio": 0.15143095548529256, "last_quartile_mean": 3.1555204145661264e-05, "mean": 4.800072150601409e-05, "normalized": [ 0.509760479963835, 0.6173326836717695, 0.7324351629463104, 0.39842116793036353, 0.4759789060227602, 0.7458666061827044, 0.6096831903000134, 0.4306170616938511, 0.4047810550951731, 0.5961537125874838, 0.6210341956779442, 0.5605157741769637, 0.27837672031341976, 0.3443893546178484, 0.25310408037089555, 1.0527443019909122, 0.7465088513707779, 1.4090569908205048, 2.369608868078584, 2.622524932193837, 3.51885122958336, 3.2980052855840127, 2.541370727154349, 1.6037570225598534, 0.918310576031736, 1.0270328185033981, 1.0005888996255676, 0.5708475165844191, 0.6850494580580762, 0.5307877571911549, 0.29761830732641337, 0.22888630579170677 ], "peak_layer": 21, "peak_normalized": 3.51885122958336, "population_cv": 0.8786708090610466, "rest_mean": 3.5768902374979075e-05, "spike_contrast": 3.1885950432011234, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011405254481360317, "top_five_layers": [ 21, 22, 20, 23, 19 ], "top_five_spike_overlap": 3 }, "values": [ 2.446887083351612e-05, 2.96324142254889e-05, 3.515741627779789e-05, 1.912450352392625e-05, 2.2847330910735764e-05, 3.580213524401188e-05, 2.9265233024489135e-05, 2.0669929654104635e-05, 1.9429782696533948e-05, 2.8615808332688175e-05, 2.9810089472448453e-05, 2.6905161575996317e-05, 1.3362283425522037e-05, 1.653093750064727e-05, 1.2149178473919164e-05, 5.0532486056908965e-05, 3.5832963476423174e-05, 6.76357522024773e-05, 0.00011374293535482138, 0.00012588308891281486, 0.00016890739789232612, 0.00015830663323868066, 0.00012198762851767242, 7.698149420320988e-05, 4.407957021612674e-05, 4.929831629851833e-05, 4.802898911293596e-05, 2.7401092665968463e-05, 3.288286825409159e-05, 2.5478195311734453e-05, 1.4285893485066481e-05, 1.0986707820848096e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.024051155894994736, "first_to_last_ratio": 0.8594772374722103, "imbalance_abs_log_ratio": 0.15143093796285173, "last_quartile_mean": 0.027983470470644534, "mean": 0.042567519849399105, "normalized": [ 0.5097604616222596, 0.6173327141338325, 0.7324351822855276, 0.3984211759638306, 0.4759788964629159, 0.7458666120674253, 0.6096832157472697, 0.430617059477717, 0.40478105545710685, 0.5961537255711045, 0.6210341986548017, 0.5605157515644588, 0.278376722109868, 0.3443893736332492, 0.25310409496395464, 1.0527443624204256, 0.7465088835832933, 1.4090569785936171, 2.3696089052435614, 2.6225247338771562, 3.5188511717225395, 3.298005336484889, 2.541370809226851, 1.6037569634689308, 0.9183106068252992, 1.0270328482800881, 1.000588834809077, 0.5708474927211314, 0.6850494767792821, 0.5307877431850209, 0.29761830682761936, 0.2288863062358959 ], "peak_layer": 21, "peak_normalized": 3.5188511717225395, "population_cv": 0.8786708002138178, "rest_mean": 0.0317202202492842, "spike_contrast": 3.1885950631853515, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10114293769001961, "top_five_layers": [ 21, 22, 20, 23, 19 ], "top_five_spike_overlap": 3 }, "values": [ 0.021699238568544388, 0.02627832256257534, 0.031177949160337448, 0.01695980131626129, 0.020261241123080254, 0.03174969181418419, 0.02595270238816738, 0.018330300226807594, 0.017230525612831116, 0.025376785546541214, 0.026435885578393936, 0.023859765380620956, 0.011849806644022465, 0.014659801498055458, 0.010774013586342335, 0.04481271654367447, 0.0317770317196846, 0.059980060905218124, 0.10086837410926819, 0.11163437366485596, 0.1497887670993805, 0.1403879076242447, 0.10817985236644745, 0.06826795637607574, 0.03909020498394966, 0.04371824115514755, 0.04259258508682251, 0.024299561977386475, 0.02916085720062256, 0.02259431779384613, 0.012668873183429241, 0.00974312238395214 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.670534108517586e-06, "first_to_last_ratio": 0.516077911622693, "imbalance_abs_log_ratio": 0.6614975333817992, "last_quartile_mean": 5.174672367047606e-06, "mean": 3.915999473136367e-06, "normalized": [ 2.91792462644838, 1.0330499050224828, 0.21324233730395303, 0.31813747476101534, 0.1980657666006576, 0.22572388219114395, 0.26853595304608496, 0.28095737396172366, 0.2790646446283966, 0.3792913632687372, 0.3694120454168363, 0.4225628265552778, 0.22944152357264785, 0.35050475469892045, 0.38580405805880497, 0.330583746531144, 0.275347554842962, 0.6571160489647494, 0.9522439259464477, 1.1118382618025555, 1.9569752952109616, 3.466111019148444, 2.5633468283111025, 2.243374425678547, 1.2633072391816056, 1.9652520232619801, 1.881284353579538, 1.3019726140072538, 1.7404895149568589, 1.1347055768273686, 0.7207866671170219, 0.5635463690963973 ], "peak_layer": 22, "peak_normalized": 3.466111019148444, "population_cv": 0.881830620778404, "rest_mean": 2.974257467013996e-06, "spike_contrast": 3.026438163483582, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.001406306197169e-06, "top_five_layers": [ 22, 1, 23, 24, 26 ], "top_five_spike_overlap": 3 }, "values": [ 1.1426591299823485e-05, 4.045422883791616e-06, 8.350568805326475e-07, 1.2458261835490703e-06, 7.756254376545257e-07, 8.83934603734815e-07, 1.051586650646641e-06, 1.100228928407887e-06, 1.0928170013357885e-06, 1.4853047787255491e-06, 1.4466173752225586e-06, 1.6547558061574819e-06, 8.984928854260943e-07, 1.372576434732764e-06, 1.5108084880921524e-06, 1.294565777243406e-06, 1.078260879694426e-06, 2.5732661015354097e-06, 3.7289867123035947e-06, 4.353958047431661e-06, 7.663514224987011e-06, 1.3573288924817462e-05, 1.0038064829132054e-05, 8.785053069004789e-06, 4.947110483044526e-06, 7.695925887674093e-06, 7.3671085374371614e-06, 5.0985240704903845e-06, 6.8157560235704295e-06, 4.4435064410208724e-06, 2.8226002086739754e-06, 2.2068472844694043e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.836107992403413e-05, "first_to_last_ratio": 0.9160653111165196, "imbalance_abs_log_ratio": 0.08766761650421023, "last_quartile_mean": 2.004341797601228e-05, "mean": 3.121470177802621e-05, "normalized": [ 0.5128089259228045, 0.6437171011700566, 0.7320107521642394, 0.42714084499329735, 0.4777939631770697, 0.7761341517210342, 0.6673269525799402, 0.4688191183265314, 0.43572863736759326, 0.6442483222440443, 0.6589268299586913, 0.5254485027294052, 0.30608252008544473, 0.35836487745274936, 0.2721971888139584, 1.0581434296571979, 0.7638096596387699, 1.3818036797996578, 2.4488923222411234, 2.174595772916911, 3.2150744119330064, 3.5033815383059985, 2.7401154537196812, 1.6705176716332228, 0.9292103758537513, 1.0277197763891293, 1.0216310132104693, 0.5115354754917107, 0.6598856618533152, 0.50319316016592, 0.28185438701748666, 0.20188752146578876 ], "peak_layer": 22, "peak_normalized": 3.5033815383059985, "population_cv": 0.8621259030079774, "rest_mean": 2.3054601317400942e-05, "spike_contrast": 3.2652589922942665, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.527924426540267e-05, "top_five_layers": [ 22, 21, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.6007177691790275e-05, 2.0093437342438847e-05, 2.2849497327115387e-05, 1.3333074093679897e-05, 1.4914196071913466e-05, 2.422679608571343e-05, 2.0830411813221872e-05, 1.463404896639986e-05, 1.3601139471575152e-05, 2.011001924984157e-05, 2.0568204490700737e-05, 1.6401718312408775e-05, 9.554274583933875e-06, 1.118625277740648e-05, 8.496554073644802e-06, 3.3029631595127285e-05, 2.3842090740799904e-05, 4.313258978072554e-05, 7.644144352525473e-05, 6.787935853935778e-05, 0.00010035758896265179, 0.00010935700993286446, 8.553188672522083e-05, 5.2144710934953764e-05, 2.9005024771322496e-05, 3.207996633136645e-05, 3.188990740454756e-05, 1.5967427316354588e-05, 2.059813414234668e-05, 1.570702443132177e-05, 8.798000635579228e-06, 6.301858775259461e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.2228491186760948e-05, "first_to_last_ratio": 0.9571529484026068, "imbalance_abs_log_ratio": 0.04379207960676997, "last_quartile_mean": 1.2775900870565238e-05, "mean": 2.0228415721135207e-05, "normalized": [ 0.4968956893086432, 0.680742973654979, 0.7082959423523417, 0.4553628765634101, 0.48416833058916253, 0.8025982815331242, 0.71544927601163, 0.49265031392139635, 0.46201741019996967, 0.6900348438458223, 0.6974262922446428, 0.46524054678094645, 0.3363013254017149, 0.39144273721666156, 0.29629939799755733, 1.053294975127128, 0.7915013073840591, 1.3752427386344863, 2.531667923112097, 1.9941217404782703, 2.7172330416452364, 3.752050564767614, 2.871442549447273, 1.6858638659668104, 0.9381697357727052, 1.0373218643217643, 1.0308781989118676, 0.4725668463395891, 0.6400815997091446, 0.4765607085370807, 0.2663935850069364, 0.19068251721593543 ], "peak_layer": 22, "peak_normalized": 3.752050564767614, "population_cv": 0.8541640127467388, "rest_mean": 1.5010413655410823e-05, "spike_contrast": 3.224802992594281, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.840562687604688e-05, "top_five_layers": [ 22, 23, 21, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.0051412573375273e-05, 1.3770351870334707e-05, 1.4327704775496386e-05, 9.211269571096636e-06, 9.793958270165604e-06, 1.623529169592075e-05, 1.4472405382548459e-05, 9.965535355149768e-06, 9.34588024392724e-06, 1.3958311683381908e-05, 1.410782897437457e-05, 9.411079190613236e-06, 6.8028430177946575e-06, 7.918266419437714e-06, 5.993667400616687e-06, 2.1306488633854315e-05, 1.601081748958677e-05, 2.7818981834570877e-05, 5.121163121657446e-05, 4.033792356494814e-05, 5.496531957760453e-05, 7.589803863083944e-05, 5.808473360957578e-05, 3.4102355130016804e-05, 1.8977687432197854e-05, 2.0983377908123657e-05, 2.0853032765444368e-05, 9.55927862378303e-06, 1.2947836694365833e-05, 9.640068128646817e-06, 5.38872018296388e-06, 3.857205228996463e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.4042374358687084e-05, "first_to_last_ratio": 0.8965146957898991, "imbalance_abs_log_ratio": 0.10924059366762352, "last_quartile_mean": 6.028052257534e-05, "mean": 9.390137262244025e-05, "normalized": [ 0.5389360174242739, 0.5974650105374263, 0.7267651782696123, 0.4014182444828728, 0.4819177901244493, 0.7732899466309664, 0.6299309978744704, 0.45445865774623606, 0.41201557220312385, 0.5831302663155696, 0.6244048764918407, 0.5807706511889962, 0.27596931886992426, 0.3314258726431575, 0.23879880407819865, 1.0945434973487749, 0.734405519454758, 1.357347632155138, 2.3148429937772614, 2.252423783537609, 3.7724800608232885, 3.3168691995820305, 2.6904907774938076, 1.6802536301987592, 0.9186543261041, 0.9881029074846317, 1.0077452263535627, 0.5222119037959454, 0.6630699433389028, 0.5491195746896886, 0.2929408036924596, 0.19380101528816393 ], "peak_layer": 21, "peak_normalized": 3.7724800608232885, "population_cv": 0.8932088546517, "rest_mean": 6.823935170056454e-05, "spike_contrast": 3.406777465012843, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002324762856005691, "top_five_layers": [ 21, 22, 23, 19, 20 ], "top_five_spike_overlap": 3 }, "values": [ 5.060683179181069e-05, 5.6102784583345056e-05, 6.824424781370908e-05, 3.769372415263206e-05, 4.525274198385887e-05, 7.26129874237813e-05, 5.915138535783626e-05, 4.267429176252335e-05, 3.868882777169347e-05, 5.475673242472112e-05, 5.863247497472912e-05, 5.45351613254752e-05, 2.5913897843565792e-05, 3.1121344363782555e-05, 2.2423535483540036e-05, 0.00010277913679601625, 6.896168633829802e-05, 0.0001274568057851866, 0.00021736693452112377, 0.0002115056850016117, 0.00035424105590209365, 0.0003114585706498474, 0.000252640777034685, 0.0001577781222295016, 8.626290218671784e-05, 9.2784219305031e-05, 9.462866000831127e-05, 4.903641456621699e-05, 6.226317782420665e-05, 5.156308179721236e-05, 2.7507543563842773e-05, 1.819818135118112e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0596221424639225, 0.07634982466697693, 0.06968431174755096, 0.11288930475711823, 0.11885104328393936, 0.1495160460472107, 0.11554427444934845, 0.13194940984249115, 0.08884616196155548, 0.10897023230791092, 0.09843935817480087, 0.09431816637516022, 0.09025302529335022, 0.0895693451166153, 0.10807885229587555, 0.2204425185918808, 0.15380074083805084, 0.08548793941736221, 0.09071414917707443, 0.10860678553581238, 0.04808543995022774, 0.05631518363952637, 0.07238366454839706, 0.08572373539209366, 0.06546958535909653, 0.06323372572660446, 0.06869969516992569, 0.07729499042034149, 0.04753003641963005, 0.06484207510948181, 0.07835857570171356, 0.08203105628490448 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.6929306190813804e-06, "first_to_last_ratio": 0.13688476606561775, "imbalance_abs_log_ratio": 1.9886158307055912, "last_quartile_mean": 2.6978390110343753e-05, "mean": 1.631737328722238e-05, "normalized": [ 0.4314194517233057, 0.19806398425123214, 0.34668833324353665, 0.22046602853167355, 0.10956900273172894, 0.12688527812920286, 0.16715726937306416, 0.21030216864948645, 0.2743958044688482, 0.24925629504156072, 0.32761471427281874, 0.40984521150785025, 0.28786819126149943, 0.29273992694234957, 0.30660983581768325, 0.24937431986069702, 0.3349491137477184, 0.5177030633789016, 0.6154108922798558, 0.7890850685582481, 3.8103077986911464, 3.428556172833401, 2.788926643404586, 2.2799755314310137, 2.2723621920706982, 2.1349169811597655, 2.003931559830236, 1.5323918909280079, 1.9997634861702531, 1.35547405261909, 1.065904631827636, 0.8620851052629044 ], "peak_layer": 21, "peak_normalized": 3.8103077986911464, "population_cv": 1.0256784776243844, "rest_mean": 1.0527649945086321e-05, "spike_contrast": 4.519705687684409, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.7581879334757106e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 7.039632237137994e-06, 3.23188396578189e-06, 5.657042947859736e-06, 3.597426484702737e-06, 1.78787831828231e-06, 2.070434447887237e-06, 2.727567562033073e-06, 3.4315789889660664e-06, 4.47741876996588e-06, 4.067208010383183e-06, 5.345811587176286e-06, 6.687597306154203e-06, 4.697252734331414e-06, 4.776746663992526e-06, 5.003067144571105e-06, 4.069133865414187e-06, 5.465489721245831e-06, 8.447554137092084e-06, 1.004188925435301e-05, 1.28757956190384e-05, 6.217421469045803e-05, 5.594503090833314e-05, 4.5507957111112773e-05, 3.7203211832093075e-05, 3.707898213178851e-05, 3.48362373188138e-05, 3.269889930379577e-05, 2.5004610506584868e-05, 3.263088728999719e-05, 2.2117776097729802e-05, 1.7392763766110875e-05, 1.406696446792921e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.4786839983571554e-05, "first_to_last_ratio": 0.1368590004352419, "imbalance_abs_log_ratio": 1.9888040770410698, "last_quartile_mean": 0.00010804433713929029, "mean": 6.53327875852483e-05, "normalized": [ 0.4303813973935663, 0.1954661274427069, 0.34854762874244455, 0.22114030543306437, 0.10883074349044422, 0.12627252893821267, 0.16836982659974983, 0.21164003420329566, 0.27785076214706256, 0.25390299121217014, 0.3323483997673342, 0.40930824768453605, 0.28788144954928885, 0.2920379121787492, 0.30710829102986115, 0.24768611608853355, 0.33445308089382414, 0.5081445956986791, 0.6162106751311494, 0.7807238268363051, 3.828274701557073, 3.4515650116091967, 2.7664213382241543, 2.2654046605337452, 2.2812415827715613, 2.1356021257971247, 2.0106696974531495, 1.537539473929636, 2.0088419319348305, 1.3422522168191928, 1.0598743605751482, 0.8540079583342102 ], "peak_layer": 21, "peak_normalized": 3.828274701557073, "population_cv": 1.0279395854576847, "rest_mean": 4.2120514451569225e-05, "spike_contrast": 4.52698797699541, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001906790625071153, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.8118016416556202e-05, 1.2770346984325442e-05, 2.2771588191972114e-05, 1.4447712601395324e-05, 7.110215847205836e-06, 8.249736310972366e-06, 1.1000070117006544e-05, 1.38270333991386e-05, 1.8152764823753387e-05, 1.6588190192123875e-05, 2.1713247406296432e-05, 2.6741248802863993e-05, 1.8808097593137063e-05, 1.9079650883213617e-05, 2.0064240743522532e-05, 1.6182024410227314e-05, 2.1850752091268077e-05, 3.3198502933373675e-05, 4.0258761146105826e-05, 5.1006863941438496e-05, 0.00025011185789480805, 0.0002255003637401387, 0.000180738017661497, 0.0001480052014812827, 0.00014903987175785005, 0.00013952484005130827, 0.00013136265624780208, 0.00010045173985417932, 0.0001312432432314381, 8.769307896727696e-05, 6.924454646650702e-05, 5.579472053796053e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.013113117485772818, "first_to_last_ratio": 0.136858997118483, "imbalance_abs_log_ratio": 1.9888041012759314, "last_quartile_mean": 0.09581480035558343, "mean": 0.05793777071812656, "normalized": [ 0.4303813960583082, 0.19546612151423692, 0.3485475910344393, 0.22114030042116126, 0.10883074113033038, 0.12627253310783348, 0.1683698287227952, 0.21164002670866952, 0.27785076861558566, 0.25390299449217174, 0.3323483845259541, 0.40930823057023097, 0.28788143760030005, 0.2920379258991919, 0.3071082822929024, 0.2476861047748367, 0.3344530552067231, 0.5081445507864877, 0.6162106424834214, 0.7807238372353658, 3.828274894940391, 3.4515650124215562, 2.7664213738142998, 2.265404688647299, 2.281241575194701, 2.135602067657764, 2.0106696442072307, 1.5375394436971075, 2.00884203416751, 1.3422522638779362, 1.0598743603945742, 0.8540078877986846 ], "peak_layer": 21, "peak_normalized": 3.828274894940391, "population_cv": 1.0279396080658147, "rest_mean": 0.03735289376022087, "spike_contrast": 4.526988119750361, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.16909610629081726, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.024935338646173477, 0.011324871331453323, 0.020194070413708687, 0.012812376022338867, 0.006305410526692867, 0.007315949071198702, 0.009754972532391548, 0.012261951342225075, 0.016098054125905037, 0.014710573479533195, 0.01925552450120449, 0.023714406415820122, 0.01667920872569084, 0.016920026391744614, 0.017793169245123863, 0.01435038074851036, 0.019377464428544044, 0.029440762475132942, 0.035701870918273926, 0.04523339867591858, 0.22180171310901642, 0.19997598230838776, 0.1602802872657776, 0.1312524974346161, 0.13217005133628845, 0.12373202294111252, 0.11649371683597565, 0.08908160775899887, 0.11638782918453217, 0.07776710391044617, 0.061406757682561874, 0.04947931319475174 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.471203732402955e-07, "first_to_last_ratio": 0.13038144854942058, "imbalance_abs_log_ratio": 2.037290905335615, "last_quartile_mean": 7.264226496772608e-06, "mean": 4.254466777098287e-06, "normalized": [ 0.436364407529321, 0.1911843551159761, 0.3504684074329156, 0.22599177156694852, 0.09379418393340402, 0.12911388991485367, 0.1538375030619483, 0.20018851917359348, 0.21440564422614256, 0.20760264501895292, 0.28595251377841346, 0.3787305694072829, 0.27369502895368186, 0.29556005516396594, 0.2777983154447535, 0.10238349267586677, 0.302592474036744, 0.3867848818041926, 0.5416819159884301, 0.6344790509535327, 4.041575473617487, 3.5020644333651707, 2.77570587266537, 2.338562686179214, 2.3623514652944007, 2.22406080211689, 2.0998230053367104, 1.589088734701869, 1.995381756812838, 1.3883763621564047, 1.1160049955561004, 0.8843947870166255 ], "peak_layer": 21, "peak_normalized": 4.041575473617487, "population_cv": 1.0776419744012766, "rest_mean": 2.6755459261780573e-06, "spike_contrast": 4.776834232976263, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2780639372067526e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.8564978745416738e-06, 8.133874871418811e-07, 1.4910561958458857e-06, 9.614744840291678e-07, 3.9904423942971334e-07, 5.493107551046705e-07, 6.54496545848815e-07, 8.516954039805569e-07, 9.121816901824786e-07, 8.832385560708644e-07, 1.2165754696980002e-06, 1.611296625014802e-06, 1.1644264077403932e-06, 1.25745043533243e-06, 1.1818837037935737e-06, 4.35587168112761e-07, 1.2873696277893032e-06, 1.6455634295198251e-06, 2.304567715327721e-06, 2.699370043046656e-06, 1.7194748579640873e-05, 1.4899416783009656e-05, 1.1809148418251425e-05, 9.949337254511192e-06, 1.0050545824924484e-05, 9.462192792852875e-06, 8.933627213991713e-06, 6.760725227650255e-06, 8.489285391988233e-06, 5.9068011069030035e-06, 4.748006176669151e-06, 3.762628239201149e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 9.853123231096106e-06, "first_to_last_ratio": 0.15080465707059237, "imbalance_abs_log_ratio": 1.89176994145415, "last_quartile_mean": 6.533699570354656e-05, "mean": 4.1971269041596315e-05, "normalized": [ 0.4681313077158978, 0.20820887504839763, 0.36906491110676237, 0.20482622912946075, 0.11627096124793933, 0.12407290917887392, 0.1787682927302214, 0.20872662343689463, 0.3073254147861055, 0.27783033377805905, 0.35982154325450194, 0.4577819740053479, 0.34235028174668086, 0.3396230936294121, 0.3348757051705037, 0.17757659165663509, 0.3534791092433193, 0.5341192315280864, 0.6428744767315707, 0.8246883036040644, 4.08439765980215, 3.623760084464356, 2.7887442899646078, 2.2190205767814533, 2.346653344716544, 2.0729802100068144, 1.8735227194091606, 1.4202102880569494, 1.8401836496959596, 1.1967241409050977, 0.93540866538392, 0.7679782020842529 ], "peak_layer": 21, "peak_normalized": 4.08439765980215, "population_cv": 1.0467961731297342, "rest_mean": 2.6329080793840586e-05, "spike_contrast": 4.8022597738792445, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012643908557947726, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.9648065062938258e-05, 8.738790711504407e-06, 1.549012267787475e-05, 8.59681676956825e-06, 4.880039796262281e-06, 5.207497451920062e-06, 7.50313211028697e-06, 8.760521268413868e-06, 1.2898837667307816e-05, 1.166089168691542e-05, 1.5102166798897088e-05, 1.9213690393371508e-05, 1.4368875781656243e-05, 1.4254412235459313e-05, 1.4055158317205496e-05, 7.453114903910318e-06, 1.483596679463517e-05, 2.2417661966755986e-05, 2.6982257622876205e-05, 3.461321466602385e-05, 0.0001714273530524224, 0.00015209380944725126, 0.00011704713688232005, 9.313510963693261e-05, 9.849201887845993e-05, 8.700561011210084e-05, 7.863412611186504e-05, 5.960802809568122e-05, 7.723484304733574e-05, 5.0228030886501074e-05, 3.926028875866905e-05, 3.2233019737759605e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 6.539388465398588e-06, "first_to_last_ratio": 0.18539523353460505, "imbalance_abs_log_ratio": 1.6852653352714542, "last_quartile_mean": 3.52726892742794e-05, "mean": 2.5340588074129755e-05, "normalized": [ 0.5326820585081189, 0.22955400672130744, 0.4075699321599326, 0.20399783458406082, 0.13335091090399903, 0.12913807236762478, 0.20439277726135907, 0.22379324161336578, 0.3549539057327596, 0.3176654165435907, 0.4067751656526541, 0.5369739555670685, 0.4337606182550087, 0.42213995757068684, 0.3963582115760412, 0.12453444551735621, 0.40318005313905786, 0.6070024892440241, 0.7294977435014097, 0.8929234728836388, 4.460419429548373, 3.910252651443441, 2.738748156484065, 2.0647804157793166, 2.3165049051088284, 1.8671237368915825, 1.5983578514794328, 1.197615503415084, 1.590133109210288, 1.0414283798969246, 0.8387268861490448, 0.6856647052905551 ], "peak_layer": 21, "peak_normalized": 4.460419429548373, "population_cv": 1.0654839953495887, "rest_mean": 1.5494638142222862e-05, "spike_contrast": 5.066831311954996, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.850871770642698e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.3498476619133726e-05, 5.817033525090665e-06, 1.0328061762265861e-05, 5.169425094209146e-06, 3.3791905025282176e-06, 3.2724346965551376e-06, 5.179433173907455e-06, 5.6710523494984955e-06, 8.994740710477345e-06, 8.049828466027975e-06, 1.0307921911589801e-05, 1.3607235814561136e-05, 1.0991749149980024e-05, 1.0697274774429388e-05, 1.0043950169347227e-05, 3.1557760848954786e-06, 1.021681964630261e-05, 1.5381800039904192e-05, 1.848590181907639e-05, 2.2627205908065662e-05, 0.00011302965140203014, 9.908810170600191e-05, 6.940148887224495e-05, 5.2322749979794025e-05, 5.870159657206386e-05, 4.731401349999942e-05, 4.0503327909391373e-05, 3.034828114323318e-05, 4.029490810353309e-05, 2.639040758367628e-05, 2.125383252860047e-05, 1.7375146853737533e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.8892148407066998e-05, "first_to_last_ratio": 0.12323172113154097, "imbalance_abs_log_ratio": 2.0936887843071057, "last_quartile_mean": 0.00023445382521458669, "mean": 0.00013867109626630736, "normalized": [ 0.40760164765243684, 0.18549021085330977, 0.32629733370345226, 0.19134578357607018, 0.0997358093357467, 0.1135317976584037, 0.15336948879180487, 0.18942939953458193, 0.2684956409479822, 0.25415697053702413, 0.3183782664979506, 0.3953176437860155, 0.26214994057141733, 0.2686491657455234, 0.2734202925021933, 0.2068336337941873, 0.30327711407815156, 0.45518103395508264, 0.5758332130814036, 0.7564185139065018, 3.831959637972952, 3.474608896989398, 2.829513018719538, 2.3332553915396175, 2.3723457642549124, 2.19075436705495, 2.160872753795178, 1.6201515958627097, 2.023759263900917, 1.3235642487571164, 1.0060145229556705, 0.8282876376878003 ], "peak_layer": 21, "peak_normalized": 3.831959637972952, "population_cv": 1.0570572109274479, "rest_mean": 8.812454328748302e-05, "spike_contrast": 4.670917624039755, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004116224823519588, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 5.652256731991656e-05, 2.572213088569697e-05, 4.524800897343084e-05, 2.6534129574429244e-05, 1.3830474017595407e-05, 1.5743578842375427e-05, 2.126791514456272e-05, 2.626838249852881e-05, 3.7232584872981533e-05, 3.5244225728092715e-05, 4.4149863242637366e-05, 5.481913103722036e-05, 3.635261964518577e-05, 3.7253874324960634e-05, 3.791549170273356e-05, 2.8681846742983907e-05, 4.2055769881699234e-05, 6.312045297818258e-05, 7.985142292454839e-05, 0.00010489338455954567, 0.0005313820438459516, 0.0004818278248421848, 0.00039237167220562696, 0.000323555083014071, 0.00032897578785195947, 0.0003037943097297102, 0.00029965059366077185, 0.00022466819791588932, 0.00028063691570423543, 0.00018354010535404086, 0.00013950513675808907, 0.00011485955474199727 ] } } } } }, "uniform_value_backward": { "bits_per_byte": 1.877417995231997, "forward": { "activation_sha256": { "attention_branch_output": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", "mlp_branch_output": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", "post_attention_state": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", "post_mlp_state": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", "pre_attention_input": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", "pre_mlp_input": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a" }, "logits_sha256": "5f6747d51014b7e09e13dda43380fde8b7ba2972518bba8957bd86ff4ed7f868", "loss_nats": 1.3013269901275635, "mixer_summary_sha256": "9abaf96810a98e69acfbc813a9b903aa15d6279ddc462a3c26f1a1846711526f" }, "loss_nats": 1.3013269901275635, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.2200722098350525, "group": 1, "latest_source_mass": 0.06159520894289017, "layer": 1, "max_source_mass": 0.9384047985076904, "mixer_index": 1, "normalized_entropy": 0.3174970857665056, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9384047985076904, "median": 0.949082612991333, "p05": 0.8598991632461548, "p95": 0.9761404991149902 }, { "label": "current_group_partial", "mean": 0.06159520894289017, "median": 0.05091731622815132, "p05": 0.02385946735739708, "p95": 0.14010077714920044 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5651201605796814, "group": 1, "latest_source_mass": 0.3442652225494385, "layer": 2, "max_source_mass": 0.6557347774505615, "mixer_index": 2, "normalized_entropy": 0.815296053174681, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.6557347774505615, "median": 0.6636689305305481, "p05": 0.3472050428390503, "p95": 0.9190996289253235 }, { "label": "current_group_partial", "mean": 0.3442652225494385, "median": 0.3363310396671295, "p05": 0.08090042322874069, "p95": 0.6527949571609497 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4018177092075348, "group": 1, "latest_source_mass": 0.14357808232307434, "layer": 2, "max_source_mass": 0.856421947479248, "mixer_index": 3, "normalized_entropy": 0.5797004164150741, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.856421947479248, "median": 0.8660546541213989, "p05": 0.7540444135665894, "p95": 0.9155591726303101 }, { "label": "current_group_partial", "mean": 0.14357808232307434, "median": 0.13394534587860107, "p05": 0.08444082736968994, "p95": 0.24595558643341064 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.687358021736145, "group": 1, "latest_source_mass": 0.5312775373458862, "layer": 3, "max_source_mass": 0.5312775373458862, "mixer_index": 4, "normalized_entropy": 0.9916480092739848, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.46872249245643616, "median": 0.4674533009529114, "p05": 0.3992259204387665, "p95": 0.5431661605834961 }, { "label": "current_group_partial", "mean": 0.5312775373458862, "median": 0.5325466990470886, "p05": 0.45683377981185913, "p95": 0.6007740497589111 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.2678314447402954, "group": 1, "latest_source_mass": 0.07946884632110596, "layer": 3, "max_source_mass": 0.920531153678894, "mixer_index": 5, "normalized_entropy": 0.38639909712095066, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.920531153678894, "median": 0.9314625263214111, "p05": 0.8444089889526367, "p95": 0.9650533199310303 }, { "label": "current_group_partial", "mean": 0.07946884632110596, "median": 0.06853749603033066, "p05": 0.03494669869542122, "p95": 0.1555909812450409 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.657238245010376, "group": 1, "latest_source_mass": 0.3777551054954529, "layer": 4, "max_source_mass": 0.6222448348999023, "mixer_index": 6, "normalized_entropy": 0.948194356759035, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.6222448348999023, "median": 0.6208515167236328, "p05": 0.5314525961875916, "p95": 0.7065955400466919 }, { "label": "current_group_partial", "mean": 0.3777551054954529, "median": 0.3791484832763672, "p05": 0.2934044897556305, "p95": 0.4685474634170532 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5044373869895935, "group": 1, "latest_source_mass": 0.2207653522491455, "layer": 4, "max_source_mass": 0.7792346477508545, "mixer_index": 7, "normalized_entropy": 0.7277493166488734, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7792346477508545, "median": 0.7866228222846985, "p05": 0.6320486664772034, "p95": 0.9385759830474854 }, { "label": "current_group_partial", "mean": 0.2207653522491455, "median": 0.21337716281414032, "p05": 0.06142403930425644, "p95": 0.367951363325119 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.650418758392334, "group": 2, "latest_source_mass": 0.4602207541465759, "layer": 5, "max_source_mass": 0.5397792458534241, "mixer_index": 8, "normalized_entropy": 0.9383559172337771, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5397792458534241, "median": 0.5213949680328369, "p05": 0.34510529041290283, "p95": 0.7807042598724365 }, { "label": "completed_group_1", "mean": 0.4602207541465759, "median": 0.4786049723625183, "p05": 0.2192957103252411, "p95": 0.6548947095870972 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0032464265823364, "group": 2, "latest_source_mass": 0.37668097019195557, "layer": 5, "max_source_mass": 0.4337552785873413, "mixer_index": 9, "normalized_entropy": 0.913194251448444, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4337552785873413, "median": 0.42766237258911133, "p05": 0.2500072717666626, "p95": 0.6592159271240234 }, { "label": "completed_group_1", "mean": 0.1895637810230255, "median": 0.18978442251682281, "p05": 0.1284591555595398, "p95": 0.25696656107902527 }, { "label": "current_group_partial", "mean": 0.37668097019195557, "median": 0.3803442120552063, "p05": 0.1759825050830841, "p95": 0.568291425704956 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0385558605194092, "group": 2, "latest_source_mass": 0.4949595034122467, "layer": 6, "max_source_mass": 0.4949595034122467, "mixer_index": 10, "normalized_entropy": 0.9453342832879565, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.24478834867477417, "median": 0.23865866661071777, "p05": 0.19890648126602173, "p95": 0.32154381275177 }, { "label": "completed_group_1", "mean": 0.26025211811065674, "median": 0.26050686836242676, "p05": 0.23401768505573273, "p95": 0.2860541343688965 }, { "label": "current_group_partial", "mean": 0.4949595034122467, "median": 0.5017017722129822, "p05": 0.42678141593933105, "p95": 0.5358227491378784 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.031773328781128, "group": 2, "latest_source_mass": 0.33116668462753296, "layer": 6, "max_source_mass": 0.40370824933052063, "mixer_index": 11, "normalized_entropy": 0.9391605568439314, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.40370824933052063, "median": 0.38690465688705444, "p05": 0.18056227266788483, "p95": 0.7100913524627686 }, { "label": "completed_group_1", "mean": 0.2651250660419464, "median": 0.2730395793914795, "p05": 0.12547528743743896, "p95": 0.35344114899635315 }, { "label": "current_group_partial", "mean": 0.33116668462753296, "median": 0.33928292989730835, "p05": 0.1502782255411148, "p95": 0.47667479515075684 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0838522911071777, "group": 2, "latest_source_mass": 0.38127952814102173, "layer": 7, "max_source_mass": 0.38127952814102173, "mixer_index": 12, "normalized_entropy": 0.9865648712351232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35612422227859497, "median": 0.36401283740997314, "p05": 0.2946957051753998, "p95": 0.39828816056251526 }, { "label": "completed_group_1", "mean": 0.2625962495803833, "median": 0.2590169608592987, "p05": 0.24180665612220764, "p95": 0.2929879426956177 }, { "label": "current_group_partial", "mean": 0.38127952814102173, "median": 0.37846946716308594, "p05": 0.3489896059036255, "p95": 0.41648316383361816 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.96053147315979, "group": 2, "latest_source_mass": 0.3146248161792755, "layer": 7, "max_source_mass": 0.5271762013435364, "mixer_index": 13, "normalized_entropy": 0.874313425279704, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5271762013435364, "median": 0.5114221572875977, "p05": 0.3548204302787781, "p95": 0.7728348970413208 }, { "label": "completed_group_1", "mean": 0.1581989824771881, "median": 0.1618250608444214, "p05": 0.07541970163583755, "p95": 0.21691158413887024 }, { "label": "current_group_partial", "mean": 0.3146248161792755, "median": 0.32365772128105164, "p05": 0.14578065276145935, "p95": 0.4483411908149719 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0704245567321777, "group": 2, "latest_source_mass": 0.38260507583618164, "layer": 8, "max_source_mass": 0.38260507583618164, "mixer_index": 14, "normalized_entropy": 0.9743424206822726, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3788670599460602, "median": 0.39538437128067017, "p05": 0.27489906549453735, "p95": 0.43731769919395447 }, { "label": "completed_group_1", "mean": 0.23852786421775818, "median": 0.2294245958328247, "p05": 0.20573674142360687, "p95": 0.29735344648361206 }, { "label": "current_group_partial", "mean": 0.38260507583618164, "median": 0.37669745087623596, "p05": 0.34335845708847046, "p95": 0.435106486082077 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9674104452133179, "group": 2, "latest_source_mass": 0.3281455934047699, "layer": 8, "max_source_mass": 0.48829177021980286, "mixer_index": 15, "normalized_entropy": 0.8805749354816949, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.48829177021980286, "median": 0.44956350326538086, "p05": 0.2823859453201294, "p95": 0.8209846615791321 }, { "label": "completed_group_1", "mean": 0.18356263637542725, "median": 0.18210050463676453, "p05": 0.05149933695793152, "p95": 0.2906108498573303 }, { "label": "current_group_partial", "mean": 0.3281455934047699, "median": 0.34843379259109497, "p05": 0.12376503646373749, "p95": 0.46937939524650574 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0832805633544922, "group": 3, "latest_source_mass": 0.31498202681541443, "layer": 9, "max_source_mass": 0.40388280153274536, "mixer_index": 16, "normalized_entropy": 0.9860444622076776, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.40388280153274536, "median": 0.403384804725647, "p05": 0.3489384353160858, "p95": 0.4601682126522064 }, { "label": "completed_group_1", "mean": 0.2811351716518402, "median": 0.28186845779418945, "p05": 0.25417307019233704, "p95": 0.30605652928352356 }, { "label": "completed_group_2", "mean": 0.31498202681541443, "median": 0.3153840899467468, "p05": 0.2613564431667328, "p95": 0.3658811151981354 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2917464971542358, "group": 3, "latest_source_mass": 0.33769795298576355, "layer": 9, "max_source_mass": 0.33769795298576355, "mixer_index": 17, "normalized_entropy": 0.9317981327650527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3349480926990509, "median": 0.3352771997451782, "p05": 0.18474003672599792, "p95": 0.48752525448799133 }, { "label": "completed_group_1", "mean": 0.1244347020983696, "median": 0.11896269023418427, "p05": 0.07986846566200256, "p95": 0.1843445897102356 }, { "label": "completed_group_2", "mean": 0.20291924476623535, "median": 0.20060715079307556, "p05": 0.1253921389579773, "p95": 0.2836313247680664 }, { "label": "current_group_partial", "mean": 0.33769795298576355, "median": 0.33908897638320923, "p05": 0.26979348063468933, "p95": 0.4016563892364502 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3533791303634644, "group": 3, "latest_source_mass": 0.28473395109176636, "layer": 10, "max_source_mass": 0.32115638256073, "mixer_index": 18, "normalized_entropy": 0.976256679908994, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.32115638256073, "median": 0.3355404734611511, "p05": 0.22348493337631226, "p95": 0.3863494396209717 }, { "label": "completed_group_1", "mean": 0.20602071285247803, "median": 0.2052205502986908, "p05": 0.1802767962217331, "p95": 0.23218519985675812 }, { "label": "completed_group_2", "mean": 0.18808895349502563, "median": 0.18706050515174866, "p05": 0.16633892059326172, "p95": 0.2130376398563385 }, { "label": "current_group_partial", "mean": 0.28473395109176636, "median": 0.275795578956604, "p05": 0.22748424112796783, "p95": 0.3660258948802948 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.308367371559143, "group": 3, "latest_source_mass": 0.29719752073287964, "layer": 10, "max_source_mass": 0.3113970160484314, "mixer_index": 19, "normalized_entropy": 0.9437875593046517, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3113970160484314, "median": 0.33913683891296387, "p05": 0.0915825292468071, "p95": 0.47554588317871094 }, { "label": "completed_group_1", "mean": 0.16179001331329346, "median": 0.16501447558403015, "p05": 0.10682816803455353, "p95": 0.20702867209911346 }, { "label": "completed_group_2", "mean": 0.2296154499053955, "median": 0.2261994481086731, "p05": 0.14510810375213623, "p95": 0.32595163583755493 }, { "label": "current_group_partial", "mean": 0.29719752073287964, "median": 0.2761377692222595, "p05": 0.194716677069664, "p95": 0.4515363574028015 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3437637090682983, "group": 3, "latest_source_mass": 0.3098427951335907, "layer": 11, "max_source_mass": 0.3217787742614746, "mixer_index": 20, "normalized_entropy": 0.969320619599697, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3217787742614746, "median": 0.31754863262176514, "p05": 0.25967735052108765, "p95": 0.40468457341194153 }, { "label": "completed_group_1", "mean": 0.1920534074306488, "median": 0.1915990114212036, "p05": 0.16391108930110931, "p95": 0.22057634592056274 }, { "label": "completed_group_2", "mean": 0.1763250231742859, "median": 0.1779440939426422, "p05": 0.14714840054512024, "p95": 0.1996835470199585 }, { "label": "current_group_partial", "mean": 0.3098427951335907, "median": 0.31089526414871216, "p05": 0.2459525167942047, "p95": 0.36900240182876587 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2280659675598145, "group": 3, "latest_source_mass": 0.29383569955825806, "layer": 11, "max_source_mass": 0.42534634470939636, "mixer_index": 21, "normalized_entropy": 0.8858623406415255, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.42534634470939636, "median": 0.4574331045150757, "p05": 0.17354995012283325, "p95": 0.6023476123809814 }, { "label": "completed_group_1", "mean": 0.14848899841308594, "median": 0.14150214195251465, "p05": 0.07509882748126984, "p95": 0.23531809449195862 }, { "label": "completed_group_2", "mean": 0.13232898712158203, "median": 0.13064970076084137, "p05": 0.07000484317541122, "p95": 0.19124652445316315 }, { "label": "current_group_partial", "mean": 0.29383569955825806, "median": 0.27858495712280273, "p05": 0.187642902135849, "p95": 0.4577033817768097 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3462648391723633, "group": 3, "latest_source_mass": 0.3101534843444824, "layer": 12, "max_source_mass": 0.3221249580383301, "mixer_index": 22, "normalized_entropy": 0.9711248035985732, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3221249580383301, "median": 0.3120178282260895, "p05": 0.2763444483280182, "p95": 0.3882659375667572 }, { "label": "completed_group_1", "mean": 0.18557845056056976, "median": 0.1851869523525238, "p05": 0.16533124446868896, "p95": 0.20681798458099365 }, { "label": "completed_group_2", "mean": 0.18214309215545654, "median": 0.18278372287750244, "p05": 0.15564095973968506, "p95": 0.2072875201702118 }, { "label": "current_group_partial", "mean": 0.3101534843444824, "median": 0.31112557649612427, "p05": 0.2608441412448883, "p95": 0.36039745807647705 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.147221326828003, "group": 3, "latest_source_mass": 0.2585829496383667, "layer": 12, "max_source_mass": 0.5053804516792297, "mixer_index": 23, "normalized_entropy": 0.8275452595084083, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5053804516792297, "median": 0.48097139596939087, "p05": 0.30125147104263306, "p95": 0.7477225065231323 }, { "label": "completed_group_1", "mean": 0.12089012563228607, "median": 0.11902710795402527, "p05": 0.05662901699542999, "p95": 0.1844676434993744 }, { "label": "completed_group_2", "mean": 0.11514648050069809, "median": 0.11994394659996033, "p05": 0.05982710421085358, "p95": 0.16130289435386658 }, { "label": "current_group_partial", "mean": 0.2585829496383667, "median": 0.25007814168930054, "p05": 0.12433226406574249, "p95": 0.42273372411727905 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3729398250579834, "group": 4, "latest_source_mass": 0.22271886467933655, "layer": 13, "max_source_mass": 0.30785465240478516, "mixer_index": 24, "normalized_entropy": 0.9903667385250569, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.30785465240478516, "median": 0.3129041790962219, "p05": 0.25217515230178833, "p95": 0.3537779450416565 }, { "label": "completed_group_1", "mean": 0.23079943656921387, "median": 0.22906282544136047, "p05": 0.19741898775100708, "p95": 0.26931890845298767 }, { "label": "completed_group_2", "mean": 0.23862707614898682, "median": 0.23815912008285522, "p05": 0.21215641498565674, "p95": 0.2663886845111847 }, { "label": "completed_group_3", "mean": 0.22271886467933655, "median": 0.22075486183166504, "p05": 0.186451256275177, "p95": 0.2638319730758667 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5232064723968506, "group": 4, "latest_source_mass": 0.24461600184440613, "layer": 13, "max_source_mass": 0.3136816620826721, "mixer_index": 25, "normalized_entropy": 0.9464213938474744, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3136816620826721, "median": 0.326117604970932, "p05": 0.20238417387008667, "p95": 0.4362232983112335 }, { "label": "completed_group_1", "mean": 0.11285123229026794, "median": 0.11373943835496902, "p05": 0.08906605839729309, "p95": 0.13395145535469055 }, { "label": "completed_group_2", "mean": 0.13196846842765808, "median": 0.1334267258644104, "p05": 0.10183953493833542, "p95": 0.16085752844810486 }, { "label": "completed_group_3", "mean": 0.19688265025615692, "median": 0.1940557211637497, "p05": 0.13830327987670898, "p95": 0.2671412229537964 }, { "label": "current_group_partial", "mean": 0.24461600184440613, "median": 0.2419893592596054, "p05": 0.18994617462158203, "p95": 0.3077915906906128 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4876129627227783, "group": 4, "latest_source_mass": 0.396677166223526, "layer": 14, "max_source_mass": 0.396677166223526, "mixer_index": 26, "normalized_entropy": 0.9243059028433878, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2035508155822754, "median": 0.20008960366249084, "p05": 0.16096048057079315, "p95": 0.255715548992157 }, { "label": "completed_group_1", "mean": 0.15372203290462494, "median": 0.15584711730480194, "p05": 0.12048184871673584, "p95": 0.18010932207107544 }, { "label": "completed_group_2", "mean": 0.13167491555213928, "median": 0.13113251328468323, "p05": 0.11609883606433868, "p95": 0.14900363981723785 }, { "label": "completed_group_3", "mean": 0.11437506973743439, "median": 0.11201842874288559, "p05": 0.09515875577926636, "p95": 0.1399364471435547 }, { "label": "current_group_partial", "mean": 0.396677166223526, "median": 0.39533013105392456, "p05": 0.3409242630004883, "p95": 0.4550648033618927 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4587602615356445, "group": 4, "latest_source_mass": 0.33461427688598633, "layer": 14, "max_source_mass": 0.33461427688598633, "mixer_index": 27, "normalized_entropy": 0.906378711639412, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28519001603126526, "median": 0.28675830364227295, "p05": 0.1500462144613266, "p95": 0.45491817593574524 }, { "label": "completed_group_1", "mean": 0.10532153397798538, "median": 0.09640030562877655, "p05": 0.06345808506011963, "p95": 0.1692449301481247 }, { "label": "completed_group_2", "mean": 0.09675674885511398, "median": 0.09718051552772522, "p05": 0.058796193450689316, "p95": 0.13363884389400482 }, { "label": "completed_group_3", "mean": 0.17811740934848785, "median": 0.1759425848722458, "p05": 0.1284305453300476, "p95": 0.23726706206798553 }, { "label": "current_group_partial", "mean": 0.33461427688598633, "median": 0.33546411991119385, "p05": 0.23162710666656494, "p95": 0.4484652876853943 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.4565696716308594, "group": 4, "latest_source_mass": 0.4221859574317932, "layer": 15, "max_source_mass": 0.4221859574317932, "mixer_index": 28, "normalized_entropy": 0.9050176216042753, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20232918858528137, "median": 0.1998670995235443, "p05": 0.15170496702194214, "p95": 0.26285088062286377 }, { "label": "completed_group_1", "mean": 0.14928826689720154, "median": 0.14749380946159363, "p05": 0.12863489985466003, "p95": 0.17874351143836975 }, { "label": "completed_group_2", "mean": 0.12029382586479187, "median": 0.11950792372226715, "p05": 0.10211217403411865, "p95": 0.14088989794254303 }, { "label": "completed_group_3", "mean": 0.1059027910232544, "median": 0.10411787033081055, "p05": 0.08704904466867447, "p95": 0.13138693571090698 }, { "label": "current_group_partial", "mean": 0.4221859574317932, "median": 0.4195709228515625, "p05": 0.35523390769958496, "p95": 0.4969034492969513 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4788098335266113, "group": 4, "latest_source_mass": 0.34753501415252686, "layer": 15, "max_source_mass": 0.34753501415252686, "mixer_index": 29, "normalized_entropy": 0.9188362111403675, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2697942852973938, "median": 0.25651901960372925, "p05": 0.146310955286026, "p95": 0.42084938287734985 }, { "label": "completed_group_1", "mean": 0.11948063224554062, "median": 0.11549928039312363, "p05": 0.0849025622010231, "p95": 0.16021209955215454 }, { "label": "completed_group_2", "mean": 0.11090938746929169, "median": 0.11184529960155487, "p05": 0.08139486610889435, "p95": 0.13886672258377075 }, { "label": "completed_group_3", "mean": 0.15228068828582764, "median": 0.15164361894130707, "p05": 0.11261995136737823, "p95": 0.19522227346897125 }, { "label": "current_group_partial", "mean": 0.34753501415252686, "median": 0.3453425168991089, "p05": 0.25332537293434143, "p95": 0.44237542152404785 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5018762350082397, "group": 4, "latest_source_mass": 0.31088152527809143, "layer": 16, "max_source_mass": 0.31088152527809143, "mixer_index": 30, "normalized_entropy": 0.9331681721954809, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.208971306681633, "median": 0.20881521701812744, "p05": 0.16508029401302338, "p95": 0.2516583502292633 }, { "label": "completed_group_1", "mean": 0.27331358194351196, "median": 0.26948875188827515, "p05": 0.23055149614810944, "p95": 0.33120542764663696 }, { "label": "completed_group_2", "mean": 0.12766750156879425, "median": 0.1261082887649536, "p05": 0.10924770683050156, "p95": 0.1494429111480713 }, { "label": "completed_group_3", "mean": 0.07916608452796936, "median": 0.07811431586742401, "p05": 0.0561223030090332, "p95": 0.10760308802127838 }, { "label": "current_group_partial", "mean": 0.31088152527809143, "median": 0.3123162090778351, "p05": 0.2707694172859192, "p95": 0.3467172384262085 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5627843141555786, "group": 4, "latest_source_mass": 0.22652018070220947, "layer": 16, "max_source_mass": 0.27452588081359863, "mixer_index": 31, "normalized_entropy": 0.9710124895666443, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27452588081359863, "median": 0.2678954601287842, "p05": 0.19414913654327393, "p95": 0.3916982412338257 }, { "label": "completed_group_1", "mean": 0.21448183059692383, "median": 0.21617943048477173, "p05": 0.17793960869312286, "p95": 0.24279263615608215 }, { "label": "completed_group_2", "mean": 0.143874391913414, "median": 0.1446641981601715, "p05": 0.11340779811143875, "p95": 0.17275942862033844 }, { "label": "completed_group_3", "mean": 0.14059773087501526, "median": 0.13659776747226715, "p05": 0.11039599776268005, "p95": 0.18301524221897125 }, { "label": "current_group_partial", "mean": 0.22652018070220947, "median": 0.22071748971939087, "p05": 0.16668984293937683, "p95": 0.293569415807724 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.580754280090332, "group": 5, "latest_source_mass": 0.24107450246810913, "layer": 17, "max_source_mass": 0.2518634498119354, "mixer_index": 32, "normalized_entropy": 0.9821778571747528, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2518634498119354, "median": 0.24593469500541687, "p05": 0.21242231130599976, "p95": 0.3016672730445862 }, { "label": "completed_group_1", "mean": 0.18758200109004974, "median": 0.1801876276731491, "p05": 0.15723729133605957, "p95": 0.24892441928386688 }, { "label": "completed_group_2", "mean": 0.13903990387916565, "median": 0.13839681446552277, "p05": 0.12513971328735352, "p95": 0.15491701662540436 }, { "label": "completed_group_3", "mean": 0.18044015765190125, "median": 0.1819951832294464, "p05": 0.14810611307621002, "p95": 0.20801132917404175 }, { "label": "completed_group_4", "mean": 0.24107450246810913, "median": 0.24384509027004242, "p05": 0.1871698796749115, "p95": 0.2886185050010681 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7564777135849, "group": 5, "latest_source_mass": 0.2205006182193756, "layer": 17, "max_source_mass": 0.2205006182193756, "mixer_index": 33, "normalized_entropy": 0.9803088772521708, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15440437197685242, "median": 0.15525281429290771, "p05": 0.11778812110424042, "p95": 0.18865147233009338 }, { "label": "completed_group_1", "mean": 0.19411899149417877, "median": 0.2002902328968048, "p05": 0.14482440054416656, "p95": 0.2256385236978531 }, { "label": "completed_group_2", "mean": 0.17806550860404968, "median": 0.1781877875328064, "p05": 0.15101563930511475, "p95": 0.20418882369995117 }, { "label": "completed_group_3", "mean": 0.14696575701236725, "median": 0.14620032906532288, "p05": 0.10836094617843628, "p95": 0.1874053031206131 }, { "label": "completed_group_4", "mean": 0.10594475269317627, "median": 0.10578415542840958, "p05": 0.08951808512210846, "p95": 0.1216079369187355 }, { "label": "current_group_partial", "mean": 0.2205006182193756, "median": 0.21638520061969757, "p05": 0.16007332503795624, "p95": 0.2944811284542084 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.5899518728256226, "group": 5, "latest_source_mass": 0.4157395362854004, "layer": 18, "max_source_mass": 0.4157395362854004, "mixer_index": 34, "normalized_entropy": 0.8873690359290373, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1636897772550583, "median": 0.16218292713165283, "p05": 0.1178273856639862, "p95": 0.21766462922096252 }, { "label": "completed_group_1", "mean": 0.11789833009243011, "median": 0.11790023744106293, "p05": 0.1032494530081749, "p95": 0.13267967104911804 }, { "label": "completed_group_2", "mean": 0.07030131667852402, "median": 0.0698043555021286, "p05": 0.06258689612150192, "p95": 0.07938572019338608 }, { "label": "completed_group_3", "mean": 0.09238425642251968, "median": 0.09013497084379196, "p05": 0.0768275037407875, "p95": 0.1164383590221405 }, { "label": "completed_group_4", "mean": 0.1399867981672287, "median": 0.1400289535522461, "p05": 0.12679682672023773, "p95": 0.15317322313785553 }, { "label": "current_group_partial", "mean": 0.4157395362854004, "median": 0.4171099066734314, "p05": 0.367803692817688, "p95": 0.4605288803577423 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.2894535064697266, "group": 5, "latest_source_mass": 0.5977407693862915, "layer": 18, "max_source_mass": 0.5977407693862915, "mixer_index": 35, "normalized_entropy": 0.7196577044045218, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1212274506688118, "median": 0.10892187058925629, "p05": 0.07249757647514343, "p95": 0.2145335078239441 }, { "label": "completed_group_1", "mean": 0.06944876164197922, "median": 0.0678764283657074, "p05": 0.05350388586521149, "p95": 0.08999023586511612 }, { "label": "completed_group_2", "mean": 0.05726535990834236, "median": 0.0559837631881237, "p05": 0.0409373864531517, "p95": 0.07825973629951477 }, { "label": "completed_group_3", "mean": 0.09096579253673553, "median": 0.08796361088752747, "p05": 0.06407731771469116, "p95": 0.12661314010620117 }, { "label": "completed_group_4", "mean": 0.06335186958312988, "median": 0.06283852458000183, "p05": 0.05032140761613846, "p95": 0.0781375840306282 }, { "label": "current_group_partial", "mean": 0.5977407693862915, "median": 0.6085352301597595, "p05": 0.4878399074077606, "p95": 0.6737192869186401 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6581614017486572, "group": 5, "latest_source_mass": 0.35773903131484985, "layer": 19, "max_source_mass": 0.35773903131484985, "mixer_index": 36, "normalized_entropy": 0.9254374988530375, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18132120370864868, "median": 0.1806161105632782, "p05": 0.1410265862941742, "p95": 0.22331592440605164 }, { "label": "completed_group_1", "mean": 0.12711825966835022, "median": 0.12722507119178772, "p05": 0.11291128396987915, "p95": 0.14178316295146942 }, { "label": "completed_group_2", "mean": 0.08560527861118317, "median": 0.08513904362916946, "p05": 0.07705061137676239, "p95": 0.09564155340194702 }, { "label": "completed_group_3", "mean": 0.09748885035514832, "median": 0.0953599065542221, "p05": 0.08346174657344818, "p95": 0.1194010004401207 }, { "label": "completed_group_4", "mean": 0.15072739124298096, "median": 0.150638610124588, "p05": 0.13762864470481873, "p95": 0.1634974181652069 }, { "label": "current_group_partial", "mean": 0.35773903131484985, "median": 0.35794761776924133, "p05": 0.32225728034973145, "p95": 0.39409786462783813 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3699336051940918, "group": 5, "latest_source_mass": 0.5385624766349792, "layer": 19, "max_source_mass": 0.5385624766349792, "mixer_index": 37, "normalized_entropy": 0.7645745027284836, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13604649901390076, "median": 0.11998516321182251, "p05": 0.06740576773881912, "p95": 0.26897257566452026 }, { "label": "completed_group_1", "mean": 0.10151474922895432, "median": 0.0991508960723877, "p05": 0.07438431680202484, "p95": 0.13600951433181763 }, { "label": "completed_group_2", "mean": 0.07715044170618057, "median": 0.0767412930727005, "p05": 0.054624155163764954, "p95": 0.09943743050098419 }, { "label": "completed_group_3", "mean": 0.11300846189260483, "median": 0.11003986746072769, "p05": 0.08064281940460205, "p95": 0.1516837179660797 }, { "label": "completed_group_4", "mean": 0.033717282116413116, "median": 0.03286907076835632, "p05": 0.024589763954281807, "p95": 0.04592139273881912 }, { "label": "current_group_partial", "mean": 0.5385624766349792, "median": 0.5415897369384766, "p05": 0.4144754409790039, "p95": 0.6506978273391724 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6926933526992798, "group": 5, "latest_source_mass": 0.3198680281639099, "layer": 20, "max_source_mass": 0.3198680281639099, "mixer_index": 38, "normalized_entropy": 0.9447101476341264, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19425281882286072, "median": 0.19403350353240967, "p05": 0.139950692653656, "p95": 0.2494640350341797 }, { "label": "completed_group_1", "mean": 0.14001017808914185, "median": 0.14058321714401245, "p05": 0.12015820294618607, "p95": 0.158815398812294 }, { "label": "completed_group_2", "mean": 0.09434760361909866, "median": 0.09424005448818207, "p05": 0.08333250135183334, "p95": 0.10548526048660278 }, { "label": "completed_group_3", "mean": 0.10331346094608307, "median": 0.10153411328792572, "p05": 0.08690286427736282, "p95": 0.12658309936523438 }, { "label": "completed_group_4", "mean": 0.14820793271064758, "median": 0.14894580841064453, "p05": 0.13278263807296753, "p95": 0.16199614107608795 }, { "label": "current_group_partial", "mean": 0.3198680281639099, "median": 0.3201481103897095, "p05": 0.2800636887550354, "p95": 0.3604968786239624 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4148362874984741, "group": 5, "latest_source_mass": 0.4996800422668457, "layer": 20, "max_source_mass": 0.4996800422668457, "mixer_index": 39, "normalized_entropy": 0.789635166883214, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16652248799800873, "median": 0.14292314648628235, "p05": 0.08186839520931244, "p95": 0.350705623626709 }, { "label": "completed_group_1", "mean": 0.10329679399728775, "median": 0.10057851672172546, "p05": 0.07478320598602295, "p95": 0.13858266174793243 }, { "label": "completed_group_2", "mean": 0.08063912391662598, "median": 0.0807541012763977, "p05": 0.056308042258024216, "p95": 0.10469165444374084 }, { "label": "completed_group_3", "mean": 0.1165856122970581, "median": 0.11373114585876465, "p05": 0.08339665830135345, "p95": 0.15619920194149017 }, { "label": "completed_group_4", "mean": 0.03327595070004463, "median": 0.032142095267772675, "p05": 0.022759035229682922, "p95": 0.04896518588066101 }, { "label": "current_group_partial", "mean": 0.4996800422668457, "median": 0.5001581907272339, "p05": 0.3463003933429718, "p95": 0.6266770362854004 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.740808129310608, "group": 6, "latest_source_mass": 0.24231041967868805, "layer": 21, "max_source_mass": 0.24231041967868805, "mixer_index": 40, "normalized_entropy": 0.9715635157550481, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21459883451461792, "median": 0.21599377691745758, "p05": 0.1570315659046173, "p95": 0.27208006381988525 }, { "label": "completed_group_1", "mean": 0.15640269219875336, "median": 0.1569681316614151, "p05": 0.13785089552402496, "p95": 0.1749057024717331 }, { "label": "completed_group_2", "mean": 0.10570268332958221, "median": 0.10564640164375305, "p05": 0.09223169088363647, "p95": 0.11907710134983063 }, { "label": "completed_group_3", "mean": 0.11097013205289841, "median": 0.10862545669078827, "p05": 0.09454305469989777, "p95": 0.13635265827178955 }, { "label": "completed_group_4", "mean": 0.17001524567604065, "median": 0.17010903358459473, "p05": 0.15088173747062683, "p95": 0.18767674267292023 }, { "label": "completed_group_5", "mean": 0.24231041967868805, "median": 0.24274763464927673, "p05": 0.2087050974369049, "p95": 0.27555277943611145 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.3179612159729004, "group": 6, "latest_source_mass": 0.603651762008667, "layer": 21, "max_source_mass": 0.603651762008667, "mixer_index": 41, "normalized_entropy": 0.6772980841960945, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08068332821130753, "median": 0.06917878985404968, "p05": 0.039396293461322784, "p95": 0.17595769464969635 }, { "label": "completed_group_1", "mean": 0.049232251942157745, "median": 0.04861311614513397, "p05": 0.03742668777704239, "p95": 0.06363335251808167 }, { "label": "completed_group_2", "mean": 0.045073024928569794, "median": 0.04482679069042206, "p05": 0.03333880379796028, "p95": 0.057495348155498505 }, { "label": "completed_group_3", "mean": 0.06111945956945419, "median": 0.059528060257434845, "p05": 0.042710162699222565, "p95": 0.08362975716590881 }, { "label": "completed_group_4", "mean": 0.028317855671048164, "median": 0.027924545109272003, "p05": 0.022549759596586227, "p95": 0.03542562201619148 }, { "label": "completed_group_5", "mean": 0.13192236423492432, "median": 0.12939827144145966, "p05": 0.10056710243225098, "p95": 0.17195004224777222 }, { "label": "current_group_partial", "mean": 0.603651762008667, "median": 0.6094026565551758, "p05": 0.5065152645111084, "p95": 0.6787958145141602 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.5319744348526, "group": 6, "latest_source_mass": 0.5132461190223694, "layer": 22, "max_source_mass": 0.5132461190223694, "mixer_index": 42, "normalized_entropy": 0.7872791226235868, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12406684458255768, "median": 0.1289612352848053, "p05": 0.06980825215578079, "p95": 0.16972985863685608 }, { "label": "completed_group_1", "mean": 0.07462044060230255, "median": 0.07392798364162445, "p05": 0.06016036123037338, "p95": 0.0917147696018219 }, { "label": "completed_group_2", "mean": 0.05378449708223343, "median": 0.053309448063373566, "p05": 0.04370884597301483, "p95": 0.06546851992607117 }, { "label": "completed_group_3", "mean": 0.05575074627995491, "median": 0.053552836179733276, "p05": 0.0437602773308754, "p95": 0.07444392144680023 }, { "label": "completed_group_4", "mean": 0.07357081025838852, "median": 0.07329098880290985, "p05": 0.06283499300479889, "p95": 0.08542977273464203 }, { "label": "completed_group_5", "mean": 0.10496053099632263, "median": 0.1038457602262497, "p05": 0.08522139489650726, "p95": 0.1291121542453766 }, { "label": "current_group_partial", "mean": 0.5132461190223694, "median": 0.5090781450271606, "p05": 0.4398295283317566, "p95": 0.59932541847229 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.2461028099060059, "group": 6, "latest_source_mass": 0.6375080347061157, "layer": 22, "max_source_mass": 0.6375080347061157, "mixer_index": 43, "normalized_entropy": 0.640370168432985, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08556011319160461, "median": 0.07947102189064026, "p05": 0.043556369841098785, "p95": 0.16172736883163452 }, { "label": "completed_group_1", "mean": 0.0446893647313118, "median": 0.04398339241743088, "p05": 0.03128381073474884, "p95": 0.05990619957447052 }, { "label": "completed_group_2", "mean": 0.04084867238998413, "median": 0.04042873531579971, "p05": 0.02942952886223793, "p95": 0.05392487719655037 }, { "label": "completed_group_3", "mean": 0.05386577919125557, "median": 0.052273523062467575, "p05": 0.033789411187171936, "p95": 0.076903335750103 }, { "label": "completed_group_4", "mean": 0.025119207799434662, "median": 0.024625632911920547, "p05": 0.019138969480991364, "p95": 0.03265690058469772 }, { "label": "completed_group_5", "mean": 0.11240880191326141, "median": 0.11003861576318741, "p05": 0.07852104306221008, "p95": 0.155371755361557 }, { "label": "current_group_partial", "mean": 0.6375080347061157, "median": 0.6392797231674194, "p05": 0.5317062735557556, "p95": 0.7386638522148132 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.6323832273483276, "group": 6, "latest_source_mass": 0.4532663822174072, "layer": 23, "max_source_mass": 0.4532663822174072, "mixer_index": 44, "normalized_entropy": 0.8388790346464895, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1418324112892151, "median": 0.14852997660636902, "p05": 0.07787702977657318, "p95": 0.1923113763332367 }, { "label": "completed_group_1", "mean": 0.08991365134716034, "median": 0.08918912708759308, "p05": 0.0752246081829071, "p95": 0.10654127597808838 }, { "label": "completed_group_2", "mean": 0.06188817322254181, "median": 0.06161077320575714, "p05": 0.04994015768170357, "p95": 0.07481888681650162 }, { "label": "completed_group_3", "mean": 0.05739111453294754, "median": 0.0548475980758667, "p05": 0.04428064078092575, "p95": 0.07816634327173233 }, { "label": "completed_group_4", "mean": 0.09848874807357788, "median": 0.09768162667751312, "p05": 0.08320866525173187, "p95": 0.11606857925653458 }, { "label": "completed_group_5", "mean": 0.0972195416688919, "median": 0.09613433480262756, "p05": 0.07767169177532196, "p95": 0.12004032731056213 }, { "label": "current_group_partial", "mean": 0.4532663822174072, "median": 0.45108917355537415, "p05": 0.37653055787086487, "p95": 0.5389904975891113 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.301480770111084, "group": 6, "latest_source_mass": 0.6134276390075684, "layer": 23, "max_source_mass": 0.6134276390075684, "mixer_index": 45, "normalized_entropy": 0.6688288103861927, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09513165056705475, "median": 0.08380547165870667, "p05": 0.055099427700042725, "p95": 0.1882638931274414 }, { "label": "completed_group_1", "mean": 0.050291214138269424, "median": 0.04947858303785324, "p05": 0.03757611662149429, "p95": 0.06516233086585999 }, { "label": "completed_group_2", "mean": 0.045042067766189575, "median": 0.0443364642560482, "p05": 0.031873252242803574, "p95": 0.06115761026740074 }, { "label": "completed_group_3", "mean": 0.05902485549449921, "median": 0.05624900758266449, "p05": 0.038259170949459076, "p95": 0.08772744238376617 }, { "label": "completed_group_4", "mean": 0.02615261636674404, "median": 0.02565642073750496, "p05": 0.01966264471411705, "p95": 0.03415588289499283 }, { "label": "completed_group_5", "mean": 0.1109299436211586, "median": 0.10745325684547424, "p05": 0.07642066478729248, "p95": 0.15984633564949036 }, { "label": "current_group_partial", "mean": 0.6134276390075684, "median": 0.6200529336929321, "p05": 0.49638131260871887, "p95": 0.7104787826538086 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.629331111907959, "group": 6, "latest_source_mass": 0.45539987087249756, "layer": 24, "max_source_mass": 0.45539987087249756, "mixer_index": 46, "normalized_entropy": 0.837310557580963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14790503680706024, "median": 0.15420395135879517, "p05": 0.08257082104682922, "p95": 0.20396333932876587 }, { "label": "completed_group_1", "mean": 0.08658000826835632, "median": 0.08578156679868698, "p05": 0.07299856841564178, "p95": 0.10278668999671936 }, { "label": "completed_group_2", "mean": 0.06608415395021439, "median": 0.06551258265972137, "p05": 0.054121918976306915, "p95": 0.07912646979093552 }, { "label": "completed_group_3", "mean": 0.06427294760942459, "median": 0.06133077293634415, "p05": 0.04914242774248123, "p95": 0.09066928923130035 }, { "label": "completed_group_4", "mean": 0.0819322019815445, "median": 0.08114765584468842, "p05": 0.06886082887649536, "p95": 0.09821749478578568 }, { "label": "completed_group_5", "mean": 0.0978257954120636, "median": 0.09643188118934631, "p05": 0.07740232348442078, "p95": 0.12209523469209671 }, { "label": "current_group_partial", "mean": 0.45539987087249756, "median": 0.45274412631988525, "p05": 0.3780696392059326, "p95": 0.5462272763252258 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.338958501815796, "group": 6, "latest_source_mass": 0.5946891903877258, "layer": 24, "max_source_mass": 0.5946891903877258, "mixer_index": 47, "normalized_entropy": 0.6880885545850224, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10803166776895523, "median": 0.09507323801517487, "p05": 0.04510519653558731, "p95": 0.2335781455039978 }, { "label": "completed_group_1", "mean": 0.0559723861515522, "median": 0.05525296553969383, "p05": 0.042058296501636505, "p95": 0.07207933813333511 }, { "label": "completed_group_2", "mean": 0.047874726355075836, "median": 0.04723185673356056, "p05": 0.03294869884848595, "p95": 0.06497423350811005 }, { "label": "completed_group_3", "mean": 0.060293328016996384, "median": 0.05662515014410019, "p05": 0.040346790105104446, "p95": 0.08978720009326935 }, { "label": "completed_group_4", "mean": 0.029215168207883835, "median": 0.028496116399765015, "p05": 0.021991699934005737, "p95": 0.039112430065870285 }, { "label": "completed_group_5", "mean": 0.10392352938652039, "median": 0.10063231736421585, "p05": 0.06900570541620255, "p95": 0.1502695530653 }, { "label": "current_group_partial", "mean": 0.5946891903877258, "median": 0.600748598575592, "p05": 0.46449366211891174, "p95": 0.7016042470932007 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7611865997314453, "group": 7, "latest_source_mass": 0.3679775297641754, "layer": 25, "max_source_mass": 0.3679775297641754, "mixer_index": 48, "normalized_entropy": 0.9050708742058073, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1573912650346756, "median": 0.15898014605045319, "p05": 0.09474922716617584, "p95": 0.2298184633255005 }, { "label": "completed_group_1", "mean": 0.10317619889974594, "median": 0.10270150750875473, "p05": 0.08812867850065231, "p95": 0.11994096636772156 }, { "label": "completed_group_2", "mean": 0.08533225208520889, "median": 0.08495655655860901, "p05": 0.07036984711885452, "p95": 0.1013721376657486 }, { "label": "completed_group_3", "mean": 0.08218386024236679, "median": 0.07942138612270355, "p05": 0.06530377268791199, "p95": 0.1088358536362648 }, { "label": "completed_group_4", "mean": 0.09052872657775879, "median": 0.08931052684783936, "p05": 0.07639768719673157, "p95": 0.10804671049118042 }, { "label": "completed_group_5", "mean": 0.11341015249490738, "median": 0.1123356968164444, "p05": 0.09177497029304504, "p95": 0.13803066313266754 }, { "label": "completed_group_6", "mean": 0.3679775297641754, "median": 0.3671194612979889, "p05": 0.2961871325969696, "p95": 0.4462817311286926 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.476489782333374, "group": 7, "latest_source_mass": 0.3396714925765991, "layer": 25, "max_source_mass": 0.3836256265640259, "mixer_index": 49, "normalized_entropy": 0.7100414956318613, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09070298075675964, "median": 0.0818760022521019, "p05": 0.04576021432876587, "p95": 0.1613437533378601 }, { "label": "completed_group_1", "mean": 0.03713828697800636, "median": 0.035560622811317444, "p05": 0.02337636426091194, "p95": 0.054472144693136215 }, { "label": "completed_group_2", "mean": 0.030013620853424072, "median": 0.028701743111014366, "p05": 0.01836019568145275, "p95": 0.04582716524600983 }, { "label": "completed_group_3", "mean": 0.03533655032515526, "median": 0.031870774924755096, "p05": 0.018502909690141678, "p95": 0.061826568096876144 }, { "label": "completed_group_4", "mean": 0.020730752497911453, "median": 0.020491186529397964, "p05": 0.014567261561751366, "p95": 0.027856238186359406 }, { "label": "completed_group_5", "mean": 0.06278066337108612, "median": 0.059403590857982635, "p05": 0.03608490154147148, "p95": 0.10122650861740112 }, { "label": "completed_group_6", "mean": 0.3836256265640259, "median": 0.3773127794265747, "p05": 0.1874689906835556, "p95": 0.5895293951034546 }, { "label": "current_group_partial", "mean": 0.3396714925765991, "median": 0.34821513295173645, "p05": 0.09212851524353027, "p95": 0.587702751159668 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7213938236236572, "group": 7, "latest_source_mass": 0.37924307584762573, "layer": 26, "max_source_mass": 0.37924307584762573, "mixer_index": 50, "normalized_entropy": 0.8278154442529139, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10584281384944916, "median": 0.10982948541641235, "p05": 0.03868259862065315, "p95": 0.16580678522586823 }, { "label": "completed_group_1", "mean": 0.06887699663639069, "median": 0.06925956159830093, "p05": 0.04082746431231499, "p95": 0.09546782076358795 }, { "label": "completed_group_2", "mean": 0.05207539349794388, "median": 0.051871635019779205, "p05": 0.035383015871047974, "p95": 0.07048650085926056 }, { "label": "completed_group_3", "mean": 0.050678279250860214, "median": 0.04831986129283905, "p05": 0.03378130868077278, "p95": 0.07452918589115143 }, { "label": "completed_group_4", "mean": 0.04768219590187073, "median": 0.04721469432115555, "p05": 0.03680850565433502, "p95": 0.060343313962221146 }, { "label": "completed_group_5", "mean": 0.06809806823730469, "median": 0.06714263558387756, "p05": 0.04759174585342407, "p95": 0.09223967790603638 }, { "label": "completed_group_6", "mean": 0.2275032103061676, "median": 0.22425106167793274, "p05": 0.14320722222328186, "p95": 0.3203037679195404 }, { "label": "current_group_partial", "mean": 0.37924307584762573, "median": 0.3716055750846863, "p05": 0.20950253307819366, "p95": 0.5737906098365784 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5405172109603882, "group": 7, "latest_source_mass": 0.4017985463142395, "layer": 26, "max_source_mass": 0.4017985463142395, "mixer_index": 51, "normalized_entropy": 0.7408321802188831, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.074287548661232, "median": 0.06164507195353508, "p05": 0.028203874826431274, "p95": 0.19368377327919006 }, { "label": "completed_group_1", "mean": 0.04491765797138214, "median": 0.04365704953670502, "p05": 0.02776508405804634, "p95": 0.06526201963424683 }, { "label": "completed_group_2", "mean": 0.035157643258571625, "median": 0.03418130800127983, "p05": 0.022541508078575134, "p95": 0.050758615136146545 }, { "label": "completed_group_3", "mean": 0.046252522617578506, "median": 0.04353209584951401, "p05": 0.026215802878141403, "p95": 0.07329559326171875 }, { "label": "completed_group_4", "mean": 0.023427214473485947, "median": 0.023055266588926315, "p05": 0.016644639894366264, "p95": 0.03150089830160141 }, { "label": "completed_group_5", "mean": 0.07749318331480026, "median": 0.07363636791706085, "p05": 0.04229677468538284, "p95": 0.12579958140850067 }, { "label": "completed_group_6", "mean": 0.29666566848754883, "median": 0.2920765280723572, "p05": 0.14379237592220306, "p95": 0.45965293049812317 }, { "label": "current_group_partial", "mean": 0.4017985463142395, "median": 0.4004032611846924, "p05": 0.18456703424453735, "p95": 0.6275562047958374 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8063910007476807, "group": 7, "latest_source_mass": 0.3437207341194153, "layer": 27, "max_source_mass": 0.3437207341194153, "mixer_index": 52, "normalized_entropy": 0.868690446228377, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11433833092451096, "median": 0.11714513599872589, "p05": 0.06115472689270973, "p95": 0.16258974373340607 }, { "label": "completed_group_1", "mean": 0.07374332845211029, "median": 0.0731307864189148, "p05": 0.05447638779878616, "p95": 0.09509895741939545 }, { "label": "completed_group_2", "mean": 0.05932536721229553, "median": 0.05929289758205414, "p05": 0.04452420026063919, "p95": 0.07498160749673843 }, { "label": "completed_group_3", "mean": 0.05389238893985748, "median": 0.051720209419727325, "p05": 0.03926049917936325, "p95": 0.07536239922046661 }, { "label": "completed_group_4", "mean": 0.06126017868518829, "median": 0.06108957529067993, "p05": 0.0506296306848526, "p95": 0.07238088548183441 }, { "label": "completed_group_5", "mean": 0.07349519431591034, "median": 0.07249070703983307, "p05": 0.05706139653921127, "p95": 0.09332700818777084 }, { "label": "completed_group_6", "mean": 0.22022445499897003, "median": 0.21959853172302246, "p05": 0.15368635952472687, "p95": 0.2904376685619354 }, { "label": "current_group_partial", "mean": 0.3437207341194153, "median": 0.33986932039260864, "p05": 0.22015926241874695, "p95": 0.4802429676055908 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.4859801530838013, "group": 7, "latest_source_mass": 0.4424910843372345, "layer": 27, "max_source_mass": 0.4424910843372345, "mixer_index": 53, "normalized_entropy": 0.7146053992378076, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08387120068073273, "median": 0.07644708454608917, "p05": 0.0386628657579422, "p95": 0.15876422822475433 }, { "label": "completed_group_1", "mean": 0.041406162083148956, "median": 0.0394824743270874, "p05": 0.025755178183317184, "p95": 0.06136181950569153 }, { "label": "completed_group_2", "mean": 0.032444946467876434, "median": 0.0307333804666996, "p05": 0.018246563151478767, "p95": 0.052875272929668427 }, { "label": "completed_group_3", "mean": 0.03686721250414848, "median": 0.03247762471437454, "p05": 0.017809681594371796, "p95": 0.06780916452407837 }, { "label": "completed_group_4", "mean": 0.025052767246961594, "median": 0.024561133235692978, "p05": 0.017057543620467186, "p95": 0.034822143614292145 }, { "label": "completed_group_5", "mean": 0.06012478470802307, "median": 0.056938059628009796, "p05": 0.03254516050219536, "p95": 0.09905251860618591 }, { "label": "completed_group_6", "mean": 0.27774184942245483, "median": 0.2747848629951477, "p05": 0.11654777824878693, "p95": 0.4562506675720215 }, { "label": "current_group_partial", "mean": 0.4424910843372345, "median": 0.4387660026550293, "p05": 0.21194897592067719, "p95": 0.6782112717628479 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.7777256965637207, "group": 7, "latest_source_mass": 0.3409850299358368, "layer": 28, "max_source_mass": 0.3409850299358368, "mixer_index": 54, "normalized_entropy": 0.8549053488311195, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10586117208003998, "median": 0.10652337223291397, "p05": 0.05095779523253441, "p95": 0.1629783660173416 }, { "label": "completed_group_1", "mean": 0.07087038457393646, "median": 0.06906798481941223, "p05": 0.04985840618610382, "p95": 0.09773125499486923 }, { "label": "completed_group_2", "mean": 0.05850498005747795, "median": 0.05767729878425598, "p05": 0.04179685190320015, "p95": 0.07779045403003693 }, { "label": "completed_group_3", "mean": 0.06360965222120285, "median": 0.06068425625562668, "p05": 0.04235687851905823, "p95": 0.09371945261955261 }, { "label": "completed_group_4", "mean": 0.04672512412071228, "median": 0.04617916792631149, "p05": 0.0356140062212944, "p95": 0.05914818495512009 }, { "label": "completed_group_5", "mean": 0.07485505938529968, "median": 0.07274073362350464, "p05": 0.052147433161735535, "p95": 0.10333258658647537 }, { "label": "completed_group_6", "mean": 0.2385886013507843, "median": 0.2345724105834961, "p05": 0.15389031171798706, "p95": 0.33735349774360657 }, { "label": "current_group_partial", "mean": 0.3409850299358368, "median": 0.33667099475860596, "p05": 0.18655306100845337, "p95": 0.5089476704597473 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6318418979644775, "group": 7, "latest_source_mass": 0.3292362689971924, "layer": 28, "max_source_mass": 0.3292362689971924, "mixer_index": 55, "normalized_entropy": 0.7847500712360619, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09244748950004578, "median": 0.06782519817352295, "p05": 0.025312144309282303, "p95": 0.24087309837341309 }, { "label": "completed_group_1", "mean": 0.05762210115790367, "median": 0.056058838963508606, "p05": 0.03864594176411629, "p95": 0.08028930425643921 }, { "label": "completed_group_2", "mean": 0.045099519193172455, "median": 0.04426075518131256, "p05": 0.03003283590078354, "p95": 0.06244165822863579 }, { "label": "completed_group_3", "mean": 0.056236449629068375, "median": 0.052902162075042725, "p05": 0.03282861411571503, "p95": 0.08620619773864746 }, { "label": "completed_group_4", "mean": 0.029373466968536377, "median": 0.02887408435344696, "p05": 0.020401213318109512, "p95": 0.04018516466021538 }, { "label": "completed_group_5", "mean": 0.07237439602613449, "median": 0.06993190199136734, "p05": 0.04555661231279373, "p95": 0.10748817771673203 }, { "label": "completed_group_6", "mean": 0.31761035323143005, "median": 0.3173876404762268, "p05": 0.16091161966323853, "p95": 0.4724525213241577 }, { "label": "current_group_partial", "mean": 0.3292362689971924, "median": 0.3128150701522827, "p05": 0.12605780363082886, "p95": 0.5847846865653992 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8334553241729736, "group": 8, "latest_source_mass": 0.270294725894928, "layer": 29, "max_source_mass": 0.27155569195747375, "mixer_index": 56, "normalized_entropy": 0.8817056346252721, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1157165914773941, "median": 0.113271564245224, "p05": 0.06316040456295013, "p95": 0.17676270008087158 }, { "label": "completed_group_1", "mean": 0.07161639630794525, "median": 0.07118864357471466, "p05": 0.04294680058956146, "p95": 0.102086141705513 }, { "label": "completed_group_2", "mean": 0.06577154994010925, "median": 0.06490545719861984, "p05": 0.04486069828271866, "p95": 0.09022088348865509 }, { "label": "completed_group_3", "mean": 0.07425849884748459, "median": 0.07179903984069824, "p05": 0.045271825045347214, "p95": 0.11117851734161377 }, { "label": "completed_group_4", "mean": 0.04881400242447853, "median": 0.04833249747753143, "p05": 0.0363147035241127, "p95": 0.0629924014210701 }, { "label": "completed_group_5", "mean": 0.08197253197431564, "median": 0.08029600977897644, "p05": 0.05680408701300621, "p95": 0.11223416775465012 }, { "label": "completed_group_6", "mean": 0.27155569195747375, "median": 0.2687028646469116, "p05": 0.18294592201709747, "p95": 0.3740781545639038 }, { "label": "completed_group_7", "mean": 0.270294725894928, "median": 0.2569413185119629, "p05": 0.13936233520507812, "p95": 0.43686625361442566 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6227548122406006, "group": 8, "latest_source_mass": 0.45173269510269165, "layer": 29, "max_source_mass": 0.45173269510269165, "mixer_index": 57, "normalized_entropy": 0.7385475426494315, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0605507455766201, "median": 0.05293343961238861, "p05": 0.02206447720527649, "p95": 0.1254502236843109 }, { "label": "completed_group_1", "mean": 0.040063515305519104, "median": 0.03803131729364395, "p05": 0.022060411050915718, "p95": 0.06445677578449249 }, { "label": "completed_group_2", "mean": 0.032619886100292206, "median": 0.030926726758480072, "p05": 0.020269516855478287, "p95": 0.050711989402770996 }, { "label": "completed_group_3", "mean": 0.03868011012673378, "median": 0.0344192236661911, "p05": 0.020861119031906128, "p95": 0.06798171997070312 }, { "label": "completed_group_4", "mean": 0.020290488377213478, "median": 0.01974661275744438, "p05": 0.014590941369533539, "p95": 0.027847949415445328 }, { "label": "completed_group_5", "mean": 0.04760073497891426, "median": 0.044880062341690063, "p05": 0.027415042743086815, "p95": 0.07675685733556747 }, { "label": "completed_group_6", "mean": 0.19551873207092285, "median": 0.18908299505710602, "p05": 0.10506269335746765, "p95": 0.30520933866500854 }, { "label": "completed_group_7", "mean": 0.11294309794902802, "median": 0.09412655979394913, "p05": 0.03902517631649971, "p95": 0.24146248400211334 }, { "label": "current_group_partial", "mean": 0.45173269510269165, "median": 0.4618796110153198, "p05": 0.24265648424625397, "p95": 0.6204620599746704 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9787883758544922, "group": 8, "latest_source_mass": 0.1984800100326538, "layer": 30, "max_source_mass": 0.24502354860305786, "mixer_index": 58, "normalized_entropy": 0.9005854004479842, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0788881778717041, "median": 0.055554650723934174, "p05": 0.03367501497268677, "p95": 0.21530622243881226 }, { "label": "completed_group_1", "mean": 0.08108963817358017, "median": 0.07848536968231201, "p05": 0.05139587074518204, "p95": 0.12043403089046478 }, { "label": "completed_group_2", "mean": 0.057054273784160614, "median": 0.056182146072387695, "p05": 0.04127398133277893, "p95": 0.07570968568325043 }, { "label": "completed_group_3", "mean": 0.08211587369441986, "median": 0.0802752673625946, "p05": 0.055154237896203995, "p95": 0.113825723528862 }, { "label": "completed_group_4", "mean": 0.026040805503726006, "median": 0.025504354387521744, "p05": 0.01917124353349209, "p95": 0.03502596169710159 }, { "label": "completed_group_5", "mean": 0.10537257045507431, "median": 0.10299044847488403, "p05": 0.07248030602931976, "p95": 0.14682361483573914 }, { "label": "completed_group_6", "mean": 0.24502354860305786, "median": 0.24406442046165466, "p05": 0.1495005488395691, "p95": 0.3413054645061493 }, { "label": "completed_group_7", "mean": 0.12593510746955872, "median": 0.11572076380252838, "p05": 0.051278628408908844, "p95": 0.2299422323703766 }, { "label": "current_group_partial", "mean": 0.1984800100326538, "median": 0.19106052815914154, "p05": 0.11614266037940979, "p95": 0.3049595355987549 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7590525150299072, "group": 8, "latest_source_mass": 0.3258843719959259, "layer": 30, "max_source_mass": 0.3258843719959259, "mixer_index": 59, "normalized_entropy": 0.800579300438408, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07131639868021011, "median": 0.052984386682510376, "p05": 0.03260878100991249, "p95": 0.19837424159049988 }, { "label": "completed_group_1", "mean": 0.044687941670417786, "median": 0.04335416108369827, "p05": 0.027018578723073006, "p95": 0.06556333601474762 }, { "label": "completed_group_2", "mean": 0.033290669322013855, "median": 0.03222261369228363, "p05": 0.01970716379582882, "p95": 0.05042369291186333 }, { "label": "completed_group_3", "mean": 0.043597716838121414, "median": 0.03997292369604111, "p05": 0.024005703628063202, "p95": 0.07244924455881119 }, { "label": "completed_group_4", "mean": 0.02658817172050476, "median": 0.02605675905942917, "p05": 0.017710059881210327, "p95": 0.03680335357785225 }, { "label": "completed_group_5", "mean": 0.06922741234302521, "median": 0.06634850800037384, "p05": 0.03838749974966049, "p95": 0.10990548133850098 }, { "label": "completed_group_6", "mean": 0.23554196953773499, "median": 0.2292930632829666, "p05": 0.10460641980171204, "p95": 0.3925905227661133 }, { "label": "completed_group_7", "mean": 0.14986534416675568, "median": 0.12543796002864838, "p05": 0.04750998318195343, "p95": 0.3270293176174164 }, { "label": "current_group_partial", "mean": 0.3258843719959259, "median": 0.31322309374809265, "p05": 0.1389845609664917, "p95": 0.5583901405334473 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9736928939819336, "group": 8, "latest_source_mass": 0.20976582169532776, "layer": 31, "max_source_mass": 0.22560220956802368, "mixer_index": 60, "normalized_entropy": 0.8982663467084998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09849467873573303, "median": 0.09779177606105804, "p05": 0.017307927832007408, "p95": 0.19556622207164764 }, { "label": "completed_group_1", "mean": 0.068942591547966, "median": 0.0661388635635376, "p05": 0.044743385165929794, "p95": 0.10368303209543228 }, { "label": "completed_group_2", "mean": 0.0599900484085083, "median": 0.05786054581403732, "p05": 0.040265996009111404, "p95": 0.08643331378698349 }, { "label": "completed_group_3", "mean": 0.06335455179214478, "median": 0.05964011698961258, "p05": 0.04099462926387787, "p95": 0.09641734510660172 }, { "label": "completed_group_4", "mean": 0.04671281576156616, "median": 0.04506188631057739, "p05": 0.03403949737548828, "p95": 0.06450185924768448 }, { "label": "completed_group_5", "mean": 0.0713595524430275, "median": 0.06962447613477707, "p05": 0.048271458595991135, "p95": 0.10105480253696442 }, { "label": "completed_group_6", "mean": 0.22560220956802368, "median": 0.22400957345962524, "p05": 0.13957585394382477, "p95": 0.31992682814598083 }, { "label": "completed_group_7", "mean": 0.1557777225971222, "median": 0.1399117112159729, "p05": 0.06281191110610962, "p95": 0.2967374622821808 }, { "label": "current_group_partial", "mean": 0.20976582169532776, "median": 0.2105630338191986, "p05": 0.08139262348413467, "p95": 0.34620192646980286 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.809161901473999, "group": 8, "latest_source_mass": 0.288146048784256, "layer": 31, "max_source_mass": 0.288146048784256, "mixer_index": 61, "normalized_entropy": 0.8233850650202157, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08854076266288757, "median": 0.08406365662813187, "p05": 0.020090684294700623, "p95": 0.1807844340801239 }, { "label": "completed_group_1", "mean": 0.05356752499938011, "median": 0.05036265030503273, "p05": 0.031541675329208374, "p95": 0.08521421998739243 }, { "label": "completed_group_2", "mean": 0.04243364930152893, "median": 0.03939494490623474, "p05": 0.023794323205947876, "p95": 0.0721740797162056 }, { "label": "completed_group_3", "mean": 0.04702243581414223, "median": 0.043501369655132294, "p05": 0.02430840954184532, "p95": 0.0788305327296257 }, { "label": "completed_group_4", "mean": 0.02648795023560524, "median": 0.025087328627705574, "p05": 0.01698113977909088, "p95": 0.040197182446718216 }, { "label": "completed_group_5", "mean": 0.05820769816637039, "median": 0.05441046878695488, "p05": 0.02834305912256241, "p95": 0.10125020891427994 }, { "label": "completed_group_6", "mean": 0.18489578366279602, "median": 0.17386877536773682, "p05": 0.07876831293106079, "p95": 0.32456493377685547 }, { "label": "completed_group_7", "mean": 0.21069815754890442, "median": 0.19004324078559875, "p05": 0.07121473550796509, "p95": 0.42417171597480774 }, { "label": "current_group_partial", "mean": 0.288146048784256, "median": 0.28367581963539124, "p05": 0.0962119847536087, "p95": 0.5029257535934448 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.8544633388519287, "group": 8, "latest_source_mass": 0.12106549739837646, "layer": 32, "max_source_mass": 0.3525030016899109, "mixer_index": 62, "normalized_entropy": 0.8440026376822011, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.07977832853794098, "median": 0.053830958902835846, "p05": 0.02079753391444683, "p95": 0.2620829939842224 }, { "label": "completed_group_1", "mean": 0.05858254432678223, "median": 0.056158941239118576, "p05": 0.03545547276735306, "p95": 0.09179478138685226 }, { "label": "completed_group_2", "mean": 0.053744811564683914, "median": 0.05185101926326752, "p05": 0.03345105051994324, "p95": 0.08065605163574219 }, { "label": "completed_group_3", "mean": 0.07626347988843918, "median": 0.07124648243188858, "p05": 0.043040163815021515, "p95": 0.12173676490783691 }, { "label": "completed_group_4", "mean": 0.031090255826711655, "median": 0.02958102896809578, "p05": 0.01873353309929371, "p95": 0.04807420074939728 }, { "label": "completed_group_5", "mean": 0.0994817242026329, "median": 0.0973888486623764, "p05": 0.0575672909617424, "p95": 0.14856761693954468 }, { "label": "completed_group_6", "mean": 0.3525030016899109, "median": 0.3572760224342346, "p05": 0.18325307965278625, "p95": 0.5024369955062866 }, { "label": "completed_group_7", "mean": 0.1274903416633606, "median": 0.1073581799864769, "p05": 0.039848171174526215, "p95": 0.2741084098815918 }, { "label": "current_group_partial", "mean": 0.12106549739837646, "median": 0.10882791876792908, "p05": 0.0503288172185421, "p95": 0.22616314888000488 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9456019401550293, "group": 8, "latest_source_mass": 0.18013209104537964, "layer": 32, "max_source_mass": 0.2524128556251526, "mixer_index": 63, "normalized_entropy": 0.885481602665194, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08823961019515991, "median": 0.07105386257171631, "p05": 0.04482517018914223, "p95": 0.1774362027645111 }, { "label": "completed_group_1", "mean": 0.06737187504768372, "median": 0.06628458946943283, "p05": 0.04582075774669647, "p95": 0.09290395677089691 }, { "label": "completed_group_2", "mean": 0.05197303742170334, "median": 0.051264435052871704, "p05": 0.035227708518505096, "p95": 0.07133325934410095 }, { "label": "completed_group_3", "mean": 0.0631948709487915, "median": 0.06016676500439644, "p05": 0.03969947248697281, "p95": 0.09776238352060318 }, { "label": "completed_group_4", "mean": 0.02739548496901989, "median": 0.026907969266176224, "p05": 0.020028207451105118, "p95": 0.036658842116594315 }, { "label": "completed_group_5", "mean": 0.08291903138160706, "median": 0.08066994696855545, "p05": 0.05307728424668312, "p95": 0.12080682814121246 }, { "label": "completed_group_6", "mean": 0.2524128556251526, "median": 0.25180506706237793, "p05": 0.14340050518512726, "p95": 0.36010295152664185 }, { "label": "completed_group_7", "mean": 0.186361163854599, "median": 0.17697101831436157, "p05": 0.08843781799077988, "p95": 0.3156980276107788 }, { "label": "current_group_partial", "mean": 0.18013209104537964, "median": 0.16958200931549072, "p05": 0.08391770720481873, "p95": 0.3105517029762268 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.596132755279541, "group": null, "latest_source_mass": 0.29075440764427185, "layer": null, "max_source_mass": 0.3670029938220978, "mixer_index": null, "normalized_entropy": 0.7264313223797062, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09668220579624176, "median": 0.1013428270816803, "p05": 0.02658359333872795, "p95": 0.15407970547676086 }, { "label": "completed_group_1", "mean": 0.037084318697452545, "median": 0.035886794328689575, "p05": 0.02348068356513977, "p95": 0.053765345364809036 }, { "label": "completed_group_2", "mean": 0.03457995876669884, "median": 0.03355112299323082, "p05": 0.019709739834070206, "p95": 0.0531856045126915 }, { "label": "completed_group_3", "mean": 0.026012206450104713, "median": 0.023950476199388504, "p05": 0.013590577989816666, "p95": 0.04525115340948105 }, { "label": "completed_group_4", "mean": 0.0258483849465847, "median": 0.02512776106595993, "p05": 0.016705092042684555, "p95": 0.03687155246734619 }, { "label": "completed_group_5", "mean": 0.037477150559425354, "median": 0.03506678715348244, "p05": 0.02071693353354931, "p95": 0.061963990330696106 }, { "label": "completed_group_6", "mean": 0.0845583900809288, "median": 0.07240844517946243, "p05": 0.03393660485744476, "p95": 0.1794772744178772 }, { "label": "completed_group_7", "mean": 0.3670029938220978, "median": 0.3608853220939636, "p05": 0.1103469580411911, "p95": 0.6465575098991394 }, { "label": "completed_group_8", "mean": 0.29075440764427185, "median": 0.28802984952926636, "p05": 0.039541684091091156, "p95": 0.5658512711524963 } ], "sources": 9 } ], "mode": "uniform_value_backward", "optimizer_state_after": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "optimizer_state_before": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.09253507107496262, 0.19514523446559906, 0.07917427271604538, 0.04215213656425476, 0.12552575767040253, 0.05027730390429497, 0.054851483553647995, 0.036243390291929245, 0.031880609691143036, 0.035054463893175125, 0.050445348024368286, 0.05600574240088463, 0.053015924990177155, 0.032114963978528976, 0.048427484929561615, 0.5742782950401306, 0.04075472429394722, 0.03505101054906845, 0.06577442586421967, 0.08553697913885117, 0.0493597537279129, 0.023477228358387947, 0.03512806445360184, 0.030729109421372414, 0.023035654798150063, 0.019568851217627525, 0.02376176416873932, 0.02176014520227909, 0.03424545377492905, 0.021422436460852623, 0.019477492198348045, 0.017491821199655533 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.125071417453e-06, "first_to_last_ratio": 2.282823561454729, "imbalance_abs_log_ratio": 0.8254130809232482, "last_quartile_mean": 3.5592200617884373e-06, "mean": 5.20536254811077e-06, "normalized": [ 1.9367384702517454, 2.0575431033491354, 1.6801075995543924, 1.4400668185251226, 1.5661035211648109, 1.3925792134893262, 1.2466333024952874, 1.1674602703335213, 1.0945737588640874, 1.02847032983817, 0.9743308590636915, 0.9218384112827168, 0.879353639681705, 0.854229397242741, 0.8405216196536197, 0.8631772058707392, 0.8729986277133472, 0.864352128283068, 0.8430011960406467, 0.8174196170838346, 0.8115146906436438, 0.7998541382616392, 0.7909603189303038, 0.7860896631748897, 0.7834851600981735, 0.7653029103464635, 0.7413896705455917, 0.7072726720787379, 0.6729563587569708, 0.6325907107306379, 0.6002222927934864, 0.5668623238577527 ], "peak_layer": 2, "peak_normalized": 2.0575431033491354, "population_cv": 0.3713778073179401, "rest_mean": 5.4035704208756755e-06, "spike_contrast": 0.7652421849089509, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1350400351802815e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.008142589853378e-05, 1.0710257811297197e-05, 8.745569175516721e-06, 7.496069883927703e-06, 8.15213661553571e-06, 7.248879683174891e-06, 6.4891783040366136e-06, 6.077053967601387e-06, 5.69765325053595e-06, 5.353560936782742e-06, 5.071745363238733e-06, 4.798503141500987e-06, 4.57735450254404e-06, 4.446573711902602e-06, 4.3752197598223574e-06, 4.493150299822446e-06, 4.544274361251155e-06, 4.499266196944518e-06, 4.388126853882568e-06, 4.254965460859239e-06, 4.224228177918121e-06, 4.16353077525855e-06, 4.117235221201554e-06, 4.091881692147581e-06, 4.078324309375603e-06, 3.983679107477656e-06, 3.859202024614206e-06, 3.681610678540892e-06, 3.502981826386531e-06, 3.2928639939200366e-06, 3.124374643448391e-06, 2.950723910544184e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.1829306408326374e-05, "first_to_last_ratio": 2.242285131401237, "imbalance_abs_log_ratio": 0.8074954938166318, "last_quartile_mean": 1.419503075794637e-05, "mean": 2.0616567582010248e-05, "normalized": [ 1.9245546228372667, 2.0377975361457814, 1.6531866271625524, 1.423879750583757, 1.538930309842627, 1.3703305332714228, 1.2383647911586764, 1.1639182408400186, 1.0938180520406116, 1.0314014172841877, 0.9796503171364493, 0.9244131813022366, 0.8827206850480096, 0.8588358130176241, 0.8455923007135348, 0.8694241468157592, 0.8793999914403602, 0.8731192864119536, 0.8549795663584978, 0.8252911344236574, 0.8191203633494749, 0.8077330232195155, 0.8000322644244665, 0.7953027221963918, 0.7913068082422922, 0.7709379699276236, 0.7460244301827129, 0.7109724829635561, 0.6762339269204858, 0.6361542658855333, 0.6048340735041462, 0.5717393653488173 ], "peak_layer": 2, "peak_normalized": 2.0377975361457814, "population_cv": 0.36204807030480873, "rest_mean": 2.1369839554676286e-05, "spike_contrast": 0.7744044538692995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6548898929613643e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.967771044699475e-05, 4.201239062240347e-05, 3.408303382457234e-05, 2.9355513106565922e-05, 3.172746073687449e-05, 2.8251512048882432e-05, 2.553083140810486e-05, 2.3995999072212726e-05, 2.255077379231807e-05, 2.1263957023620605e-05, 2.019702696998138e-05, 1.9058226826018654e-05, 1.8198670659330674e-05, 1.7706246580928564e-05, 1.743321081448812e-05, 1.79245416802587e-05, 1.8130209355149418e-05, 1.8000722775468603e-05, 1.7626744011067785e-05, 1.7014670447679237e-05, 1.688745032879524e-05, 1.6652682461426593e-05, 1.6493919247295707e-05, 1.6396412320318632e-05, 1.631403029023204e-05, 1.5894094758550636e-05, 1.5380463082692586e-05, 1.4657812243967783e-05, 1.3941622455604374e-05, 1.3115317415213212e-05, 1.2469602552300785e-05, 1.1787303265009541e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028226546943187714, "first_to_last_ratio": 2.242285122589081, "imbalance_abs_log_ratio": 0.8074954898866428, "last_quartile_mean": 0.01258829515427351, "mean": 0.018282978096976876, "normalized": [ 1.924554554381315, 2.0377975654135123, 1.6531866056622397, 1.4238797675022206, 1.5389303095647766, 1.3703305145930151, 1.2383648563072303, 1.1639182544549975, 1.0938181015007578, 1.0314014326659862, 0.9796503379512251, 0.9244131595482207, 0.8827206459493627, 0.8588358074372765, 0.8455922449868795, 0.8694241575396449, 0.8794000131907383, 0.8731192962202396, 0.854979598609066, 0.8252910899109279, 0.8191203509444464, 0.8077330175197024, 0.8000322649688487, 0.795302701402855, 0.79130681694379, 0.7709379635040872, 0.7460244053110716, 0.7109724950182295, 0.6762339125172862, 0.6361542907855866, 0.6048340936091356, 0.5717393740853287 ], "peak_layer": 2, "peak_normalized": 2.0377975654135123, "population_cv": 0.36204806883319857, "rest_mean": 0.018950987227812962, "spike_contrast": 0.7744044473273423, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.014675728790462018, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.035186588764190674, 0.037257008254528046, 0.030225174501538277, 0.026032762601971626, 0.028136229142546654, 0.02505372278392315, 0.022640997543931007, 0.021279891952872276, 0.019998252391815186, 0.018857089802622795, 0.017910925671458244, 0.01690102554857731, 0.01613876223564148, 0.015702076256275177, 0.015459944494068623, 0.0158956628292799, 0.016078051179647446, 0.015963220968842506, 0.015631573274731636, 0.015088778920471668, 0.014975959435105324, 0.014767765067517757, 0.014626972377300262, 0.014540501870214939, 0.014467445202171803, 0.014095041900873184, 0.013639547862112522, 0.01299869455397129, 0.012363569810986519, 0.01163079496473074, 0.011058168485760689, 0.010453098453581333 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.3494145605363883e-06, "first_to_last_ratio": 1.5177010929884052, "imbalance_abs_log_ratio": 0.4171967511446403, "last_quartile_mean": 8.891174729797058e-07, "mean": 1.0250130149103143e-06, "normalized": [ 2.738596187346452, 1.4794765430196648, 1.3169609982966792, 1.1024070457743096, 1.0148842654352301, 0.9797975144938003, 0.96217162006378, 0.9375881795084541, 0.9109044983437103, 0.8889313853518959, 0.900009111988326, 0.9067663499946724, 0.8764724089040279, 0.8712282957663963, 0.8670476678993289, 0.8512105160986065, 0.8567132773721255, 0.8738651868927485, 0.8997173009911049, 0.9374637355940177, 0.949227789273703, 0.9688228256439222, 0.9778524405833988, 0.9925195201244542, 0.9855597557475544, 0.9661836608096859, 0.9353656023236144, 0.8926044217979134, 0.8523421571494035, 0.800136382267235, 0.7757454859502885, 0.7314278691934953 ], "peak_layer": 1, "peak_normalized": 2.738596187346452, "population_cv": 0.3416815162600759, "rest_mean": 1.0297970797418437e-06, "spike_contrast": 0.9702679144036179, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.99179064820055e-07, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.8070967346138787e-06, 1.516482711849676e-06, 1.3499021633833763e-06, 1.129981569647498e-06, 1.040269580698805e-06, 1.0043052043329226e-06, 9.862384331427165e-07, 9.610400866222335e-07, 9.33688966142654e-07, 9.111662393479492e-07, 9.225210533259087e-07, 9.294473102272605e-07, 8.983956263364234e-07, 8.930203421186889e-07, 8.88735144144448e-07, 8.725018574295973e-07, 8.781422593528987e-07, 8.957231898421014e-07, 9.222219432558632e-07, 9.609125299903098e-07, 9.729708381200908e-07, 9.930560054272064e-07, 1.0023114782597986e-06, 1.0173454256801051e-06, 1.0102115766130737e-06, 9.903508271236205e-07, 9.5876191608113e-07, 9.149311495093571e-07, 8.736618042348709e-07, 8.201502055271703e-07, 7.95149219356972e-07, 7.497230853914516e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.385878633504035e-05, "first_to_last_ratio": 2.4650377133645693, "imbalance_abs_log_ratio": 0.9022071069169957, "last_quartile_mean": 9.678872743279499e-06, "mean": 1.5052104913593212e-05, "normalized": [ 1.9687527575116004, 2.0294475105176777, 1.6968710846545252, 1.4810263904268979, 1.5821534963273902, 1.4269282679347157, 1.2874882094723543, 1.207970092684867, 1.1337626821930848, 1.062868269769312, 1.0013238472605475, 0.9383693915051101, 0.893007722864056, 0.8652298987199482, 0.8502729480950922, 0.8889049949994228, 0.902010159160841, 0.8913150303986681, 0.8591568947108383, 0.8064459655370104, 0.7966210501283933, 0.7772027605950662, 0.7602554733595914, 0.748418831566216, 0.7418869141825569, 0.7197374405224362, 0.6919515801072532, 0.6600392455527566, 0.631139243766499, 0.594431491034554, 0.565796746616864, 0.5392136078238537 ], "peak_layer": 2, "peak_normalized": 2.0294475105176777, "population_cv": 0.38729311327642874, "rest_mean": 1.5707493056888104e-05, "spike_contrast": 0.7329628539770106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1513008939800783e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.9633873054990545e-05, 3.0547456844942644e-05, 2.554148159106262e-05, 2.2292564608505927e-05, 2.381474041612819e-05, 2.1478273993125185e-05, 1.937940760399215e-05, 1.8182492567575537e-05, 1.706551483948715e-05, 1.5998404705896974e-05, 1.5072031601448543e-05, 1.412443452863954e-05, 1.3441645933198743e-05, 1.3023531209910288e-05, 1.2798397619917523e-05, 1.337989124294836e-05, 1.3577151548815891e-05, 1.3416167348623276e-05, 1.2932119716424495e-05, 1.2138709280407056e-05, 1.1990823622909375e-05, 1.1698537491611205e-05, 1.144344514614204e-05, 1.126527877204353e-05, 1.1166959666297771e-05, 1.0833563464984763e-05, 1.0415327778900973e-05, 9.934979971149005e-06, 9.499974112259224e-06, 8.947445166995749e-06, 8.516431989846751e-06, 8.116299795801751e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.751610693645489e-05, "first_to_last_ratio": 2.3594114654247895, "imbalance_abs_log_ratio": 0.8584122088800984, "last_quartile_mean": 7.423930583172478e-06, "mean": 1.1241273455198098e-05, "normalized": [ 1.9313123081075616, 1.9532131132782136, 1.6584498227621727, 1.458759062451573, 1.5411339282816632, 1.4256617364160402, 1.287861390019934, 1.2091763566215845, 1.1379116841730463, 1.0621022378529084, 0.9972026598452313, 0.9362662172108906, 0.8866971704599771, 0.8635601104202946, 0.8515524995644437, 0.9050254579606748, 0.9262405785753693, 0.9113277687772972, 0.8653429711725167, 0.8112155582189827, 0.8037618636954775, 0.7801120976934229, 0.763059385733637, 0.7497162474482649, 0.7450633821823573, 0.7253917192991086, 0.6970996868130734, 0.6708244568956084, 0.6493864786868861, 0.6191431780854164, 0.5992363193314693, 0.5771925519649024 ], "peak_layer": 2, "peak_normalized": 1.9532131132782136, "population_cv": 0.367876005198269, "rest_mean": 1.172351868363977e-05, "spike_contrast": 0.7367369349328763, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.637149221613072e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.17104097828269e-05, 2.1956602722639218e-05, 1.86430879693944e-05, 1.639830952626653e-05, 1.732430791889783e-05, 1.602625343366526e-05, 1.447720205760561e-05, 1.3592682080343366e-05, 1.2791576409654226e-05, 1.1939381693082396e-05, 1.1209827789571136e-05, 1.0524824574531522e-05, 9.967605365091003e-06, 9.707515346235596e-06, 9.57253450906137e-06, 1.0173638656851836e-05, 1.0412123629066627e-05, 1.024448465614114e-05, 9.727556971483864e-06, 9.119095921050757e-06, 9.035306902660523e-06, 8.76945341587998e-06, 8.5777592175873e-06, 8.427765351370908e-06, 8.375461220566649e-06, 8.15432667877758e-06, 7.83628820499871e-06, 7.540921160398284e-06, 7.2999309850274585e-06, 6.95995777277858e-06, 6.736179329891456e-06, 6.488379312941106e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.572680194949498e-05, "first_to_last_ratio": 2.1392798428041573, "imbalance_abs_log_ratio": 0.760469250343375, "last_quartile_mean": 3.072379809054837e-05, "mean": 4.3481031354986044e-05, "normalized": [ 1.9052658835371106, 1.9135764766323078, 1.63425226196895, 1.412889347119181, 1.4908784667860193, 1.3457175192541695, 1.2381709412398325, 1.1522100846113699, 1.1016927428728427, 1.0254980546866814, 0.9643692448719049, 0.9205305483556507, 0.884025033242643, 0.8683067103948131, 0.8527072800323677, 0.8631028842013038, 0.8693948150903978, 0.864259931478139, 0.8658847675556672, 0.8471741358779341, 0.8431650913109492, 0.8356030774323144, 0.8293172543204428, 0.8191888026538522, 0.8215172881346767, 0.7971178058174285, 0.7748791379460219, 0.7321907952521126, 0.7000071600353317, 0.6525819851008854, 0.6088054122091195, 0.5657190599775788 ], "peak_layer": 2, "peak_normalized": 1.9135764766323078, "population_cv": 0.33981080750369574, "rest_mean": 4.485182479483127e-05, "spike_contrast": 0.8043986380678882, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6078746779821814e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.284292562166229e-05, 8.32042787806131e-05, 7.105897384462878e-05, 6.143388600321487e-05, 6.482493336079642e-05, 5.8513185649644583e-05, 5.383694951888174e-05, 5.00992828165181e-05, 4.790273669641465e-05, 4.458971307030879e-05, 4.193176937405951e-05, 4.0025617636274546e-05, 3.843832018901594e-05, 3.7754871300421655e-05, 3.707659197971225e-05, 3.752860357053578e-05, 3.780218321480788e-05, 3.757891317945905e-05, 3.764956272789277e-05, 3.683600516524166e-05, 3.666168777272105e-05, 3.6332883610157296e-05, 3.605956953833811e-05, 3.561917401384562e-05, 3.5720418964046985e-05, 3.4659504308365285e-05, 3.3692544093355536e-05, 3.1836410926189274e-05, 3.043703327421099e-05, 2.8374937755870633e-05, 2.647148721734993e-05, 2.4598048184998333e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.116500623524189, 0.0468219518661499, 0.09375449270009995, 0.041188232600688934, 0.024237047880887985, 0.022429833188652992, 0.03027583472430706, 0.04546776041388512, 0.030174262821674347, 0.03663193807005882, 0.05748353153467178, 0.10596779733896255, 0.05394246056675911, 0.06537146866321564, 0.08060523867607117, 0.06690707802772522, 0.033449217677116394, 0.022565271705389023, 0.036697570234537125, 0.07367803901433945, 0.042231619358062744, 0.052135858684778214, 0.06269875913858414, 0.0728035643696785, 0.051585447043180466, 0.04862172156572342, 0.05972515419125557, 0.06787826865911484, 0.06439156830310822, 0.07232685387134552, 0.09627991914749146, 0.1271906942129135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.901735841642221e-06, "first_to_last_ratio": 2.319841868234872, "imbalance_abs_log_ratio": 0.8414990231116138, "last_quartile_mean": 3.40615278560108e-06, "mean": 5.0787282575015524e-06, "normalized": [ 1.8471312527953234, 2.083119522653031, 1.6411120697088142, 1.4465389141337504, 1.6030351711912503, 1.4179806134683028, 1.2455229358727704, 1.1623539789603097, 1.087561721245063, 1.0281126130561575, 0.9635841018330411, 0.9122839040717997, 0.8813528037235394, 0.8604108318814777, 0.8561651321622351, 0.8887436718072528, 0.8917236395900334, 0.8870579095571277, 0.8597154672649976, 0.8329428550551852, 0.8190933157744664, 0.8104435207647023, 0.8055448086648276, 0.8031059287493388, 0.7839217090043826, 0.7606298596198403, 0.7253003945161415, 0.6892581493416868, 0.6485348180910961, 0.6152321716145727, 0.5812511282777914, 0.5612350855496906 ], "peak_layer": 2, "peak_normalized": 2.083119522653031, "population_cv": 0.3684028416393891, "rest_mean": 5.262670524812334e-06, "spike_contrast": 0.7763054887744506, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.085440014023334e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 9.381077688885853e-06, 1.0579597983451094e-05, 8.334762242157012e-06, 7.34657805878669e-06, 8.141380021697842e-06, 7.201538210210856e-06, 6.325672529783333e-06, 5.90327999816509e-06, 5.523430445464328e-06, 5.221504579822067e-06, 4.893781806458719e-06, 4.6332420424732845e-06, 4.476151389098959e-06, 4.369792804936878e-06, 4.348230049799895e-06, 4.5136875996831805e-06, 4.528822046268033e-06, 4.505126071308041e-06, 4.366261237009894e-06, 4.230290414852789e-06, 4.159952368354425e-06, 4.11602241001674e-06, 4.091143182449741e-06, 4.078756774106296e-06, 3.981325335189467e-06, 3.863032361550722e-06, 3.683603608806152e-06, 3.50055483977485e-06, 3.293732106612879e-06, 3.1245970149029745e-06, 2.952016529889079e-06, 2.850360488082515e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.0931253377275425e-05, "first_to_last_ratio": 2.2774504398256985, "imbalance_abs_log_ratio": 0.8230565891361505, "last_quartile_mean": 1.3581526445705094e-05, "mean": 2.0111286517021654e-05, "normalized": [ 1.8305147349095139, 2.0638827488953453, 1.6144583041246288, 1.4292024216268573, 1.5742414021575613, 1.3951965505267199, 1.2383780380531688, 1.1581634757031298, 1.0870733067057088, 1.0303290805210676, 0.9686529136024508, 0.9150769272896286, 0.8858327771787193, 0.8660099552886569, 0.8620450653043313, 0.8950473453908027, 0.8985730286373522, 0.8961938413974023, 0.8726197634428304, 0.8416466438098196, 0.8273537929153006, 0.8203765923046102, 0.8150971572776908, 0.8114850979119946, 0.7901082294468773, 0.7656064448388112, 0.7288917101067979, 0.6928620148788858, 0.6522549280133905, 0.6198972579804878, 0.5863035049818077, 0.5666249447776498 ], "peak_layer": 2, "peak_normalized": 2.0638827488953453, "population_cv": 0.3585662940639587, "rest_mean": 2.0808164292786926e-05, "spike_contrast": 0.7856602003837604, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6348146527889184e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.6814006307395175e-05, 4.1507337300572544e-05, 3.246883352403529e-05, 2.874309939215891e-05, 3.1660019885748625e-05, 2.8059197575203143e-05, 2.4905375539674424e-05, 2.329215749341529e-05, 2.1862442736164667e-05, 2.0721243345178664e-05, 1.948085628100671e-05, 1.8403374269837514e-05, 1.7815236788010225e-05, 1.741657433740329e-05, 1.733683529892005e-05, 1.8000553609454073e-05, 1.8071459635393694e-05, 1.802361111913342e-05, 1.754950608301442e-05, 1.692659679974895e-05, 1.6639149180264212e-05, 1.6498828699695878e-05, 1.6392652469221503e-05, 1.6320009308401495e-05, 1.5890092981862836e-05, 1.5397330571431667e-05, 1.46589500218397e-05, 1.3934346497990191e-05, 1.3117685739416629e-05, 1.2466931366361678e-05, 1.1791317774623167e-05, 1.1395556612114888e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027430144604295492, "first_to_last_ratio": 2.2774504162405003, "imbalance_abs_log_ratio": 0.8230565787801849, "last_quartile_mean": 0.012044233502820134, "mean": 0.017834889993537217, "normalized": [ 1.8305147145419467, 2.0638826670965, 1.6144583460455657, 1.429202391478539, 1.5742413594714617, 1.3951965580068768, 1.2383779969546256, 1.1581635530608791, 1.087073254641334, 1.0303291198940532, 0.9686528876006529, 0.9150769923697103, 0.8858327087407725, 0.866009958574656, 0.8620451161142623, 0.8950472977612934, 0.8985730294936267, 0.8961938211463923, 0.8726198047926574, 0.8416466499561293, 0.8273538542424902, 0.8203766463914877, 0.8150971349229298, 0.8114850849561472, 0.7901082410359804, 0.7656064994836816, 0.7288916938108543, 0.6928620554424487, 0.6522549396913511, 0.6198972466405528, 0.5863034743836752, 0.5666249012564666 ], "peak_layer": 2, "peak_normalized": 2.0638826670965, "population_cv": 0.3585662833971213, "rest_mean": 0.01845288811320508, "spike_contrast": 0.7856602206868666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.014497700147330761, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.0326470285654068, 0.03680912032723427, 0.028793687000870705, 0.025489667430520058, 0.028076421469449997, 0.024883177131414413, 0.022086335346102715, 0.020655719563364983, 0.019387831911444664, 0.018375806510448456, 0.017275817692279816, 0.016320297494530678, 0.01579872891306877, 0.015445192344486713, 0.015374479815363884, 0.01596307009458542, 0.016025951132178307, 0.015983518213033676, 0.015563078224658966, 0.01501067541539669, 0.014755764976143837, 0.014631327241659164, 0.014537167735397816, 0.014472747221589088, 0.0140914935618639, 0.01365450769662857, 0.0129997031763196, 0.012357118539512157, 0.011632895097136497, 0.011055799201130867, 0.010456657968461514, 0.010105692781507969 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.291231697564399e-06, "first_to_last_ratio": 1.5202661658451317, "imbalance_abs_log_ratio": 0.41888542863701395, "last_quartile_mean": 8.493458096836548e-07, "mean": 9.93008823613195e-07, "normalized": [ 2.660433833539824, 1.4760353050092347, 1.1979770584425742, 1.0900493665572268, 1.0508641630009479, 1.017485294217113, 0.969866960695177, 0.9398678696701823, 0.9117101913843627, 0.9035984270963272, 0.9003870600648762, 0.9006127144113643, 0.88474134857999, 0.8793888984633811, 0.8796998458023265, 0.8766255199949985, 0.8797862836671659, 0.8997429548627749, 0.9211914520157214, 0.9587231163907084, 0.9600152766046289, 0.9836978204801501, 1.0020995832501742, 1.0127953245043364, 0.9873250051607358, 0.9639498022761569, 0.915477107739169, 0.8777702206461201, 0.8263790746766105, 0.7874102520005254, 0.7530783342023123, 0.731214534592803 ], "peak_layer": 1, "peak_normalized": 2.660433833539824, "population_cv": 0.3259281809459305, "rest_mean": 9.949973050256463e-07, "spike_contrast": 0.9872097331566548, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.82271023985959e-07, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.6418342713441234e-06, 1.4657160818387638e-06, 1.1896017895196564e-06, 1.0824286391653004e-06, 1.043517386278836e-06, 1.0103718750542612e-06, 9.630864497012226e-07, 9.332970876130275e-07, 9.05336264622747e-07, 8.972812111096573e-07, 8.94092295311566e-07, 8.943163720687153e-07, 8.785559657553677e-07, 8.732409355616255e-07, 8.735497090128774e-07, 8.704968763595389e-07, 8.736355425753572e-07, 8.934526931625442e-07, 9.147512400886626e-07, 9.520205139779137e-07, 9.533036404718587e-07, 9.768206155058579e-07, 9.950937283065286e-07, 1.0057146937469952e-06, 9.80422441898554e-07, 9.572106591804186e-07, 9.090768458008824e-07, 8.716335742064985e-07, 8.206017128031817e-07, 7.819053280400112e-07, 7.478134307348228e-07, 7.261024848048692e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.325468449271284e-05, "first_to_last_ratio": 2.49752403690097, "imbalance_abs_log_ratio": 0.9152998558790303, "last_quartile_mean": 9.31109536850272e-06, "mean": 1.4726752056049008e-05, "normalized": [ 1.8853975232156222, 2.0473116740268433, 1.6582395075614986, 1.4857492998515238, 1.6139665585544047, 1.450824782288974, 1.2878281373152924, 1.2033034259685012, 1.1275524261509264, 1.0620148402767218, 0.9899925412551716, 0.9287541765485573, 0.8955670967012345, 0.8710703550878681, 0.8665017455395924, 0.9137558765532069, 0.9194683059544567, 0.9140179774903064, 0.8758167041500393, 0.8223153812435762, 0.8027710133686686, 0.7865201376885268, 0.7720207805468117, 0.7611819433729968, 0.737807344555837, 0.7143323889943822, 0.6774046890577128, 0.6461902213762475, 0.6100577174023712, 0.5819822851664526, 0.5536782248275314, 0.5366049179081446 ], "peak_layer": 2, "peak_normalized": 2.0473116740268433, "population_cv": 0.38353466933491126, "rest_mean": 1.5348383958216464e-05, "spike_contrast": 0.7407906796766099, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1369939784344751e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.776578185148537e-05, 3.015025140484795e-05, 2.4420482077402994e-05, 2.1880261556361802e-05, 2.376848533458542e-05, 2.1365936845541e-05, 1.8965525669045746e-05, 1.772075120243244e-05, 1.6605185010121204e-05, 1.5640029232599773e-05, 1.457937469240278e-05, 1.367753247905057e-05, 1.3188794582674745e-05, 1.2828037142753601e-05, 1.2760756362695247e-05, 1.3456656233756803e-05, 1.3540781765186694e-05, 1.3460516129271127e-05, 1.2897935448563658e-05, 1.211003473144956e-05, 1.1822209671663586e-05, 1.1582887054828461e-05, 1.1369358617230318e-05, 1.120973774959566e-05, 1.086550582840573e-05, 1.0519795978325419e-05, 9.975970897357911e-06, 9.516283171251416e-06, 8.984168744063936e-06, 8.570708814659156e-06, 8.153881935868412e-06, 7.902447578089777e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.711517734293011e-05, "first_to_last_ratio": 2.384061847169584, "imbalance_abs_log_ratio": 0.8688056914712909, "last_quartile_mean": 7.178998885137844e-06, "mean": 1.102206732639388e-05, "normalized": [ 1.8531196998586799, 1.963921090780072, 1.6275918064471853, 1.4630803862888466, 1.5667065333158392, 1.442954782698019, 1.2970011659327807, 1.2081052444056364, 1.1324273638297324, 1.0585415613956308, 0.9863531089650689, 0.9268820741278728, 0.8902461316863771, 0.8668001748241623, 0.8666116262214356, 0.9296576580884496, 0.9423272990336848, 0.9350798538143901, 0.8819279614478344, 0.8237566329745545, 0.8069480820231186, 0.7863655059325534, 0.7701915014821978, 0.7627658221351614, 0.7427791751513323, 0.714459505084982, 0.683554697524179, 0.6634891654356642, 0.6327577648283097, 0.612919811315865, 0.5879751612383226, 0.5727016517120618 ], "peak_layer": 2, "peak_normalized": 1.963921090780072, "population_cv": 0.36382703407118977, "rest_mean": 1.1483749033316229e-05, "spike_contrast": 0.7427004965250654, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.528986109013204e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.042521009570919e-05, 2.164647048630286e-05, 1.7939426470547915e-05, 1.6126170521602035e-05, 1.7268344890908338e-05, 1.5904344763839617e-05, 1.4295634173322469e-05, 1.3315817341208458e-05, 1.2481690646382049e-05, 1.1667316357488744e-05, 1.0871650374610908e-05, 1.0216156624665018e-05, 9.812352800508961e-06, 9.553929885441903e-06, 9.551851690048352e-06, 1.0246749297948554e-05, 1.0386394933448173e-05, 1.0306513104296755e-05, 9.720669368107338e-06, 9.079501069209073e-06, 8.894236088963225e-06, 8.66737354954239e-06, 8.489102583553176e-06, 8.407256245845929e-06, 8.186962077161297e-06, 7.874820767028723e-06, 7.5341858973843046e-06, 7.313022251764778e-06, 6.9742986852361355e-06, 6.755643426004099e-06, 6.48070181341609e-06, 6.312356163107324e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.3685856275697e-05, "first_to_last_ratio": 2.1822227741405626, "imbalance_abs_log_ratio": 0.780343978505805, "last_quartile_mean": 2.918393897743954e-05, "mean": 4.2313938195093215e-05, "normalized": [ 1.7787137281139571, 1.9558551999471387, 1.6050111202233173, 1.4144048771289366, 1.5329849841630987, 1.3719415533754096, 1.229370474014663, 1.15235696212458, 1.0910654327049436, 1.0293903648071914, 0.9556505571608248, 0.9144304448831003, 0.8945689808825378, 0.8756421608621586, 0.8706776546209917, 0.8937649342904419, 0.8888623306953729, 0.8872279288865894, 0.8842406103569739, 0.8655423142717734, 0.8568957182744374, 0.8484542607681987, 0.8405793844255519, 0.8447650350125491, 0.8186152079032946, 0.7955630923758639, 0.7545060763754283, 0.7184499344207425, 0.6699909954890287, 0.6249915977031716, 0.5822893469308854, 0.5531967368068471 ], "peak_layer": 2, "peak_normalized": 1.9558551999471387, "population_cv": 0.3361774415143179, "rest_mean": 4.355309465609025e-05, "spike_contrast": 0.8179095788025235, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5622493305709214e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.526438275817782e-05, 8.275993604911491e-05, 6.791434134356678e-05, 5.984904055367224e-05, 6.486663187388331e-05, 5.8052250096807256e-05, 5.2019506256328896e-05, 4.8760761274024844e-05, 4.616727528627962e-05, 4.355756027507596e-05, 4.043733861180954e-05, 3.86931533284951e-05, 3.785273656831123e-05, 3.705186827573925e-05, 3.6841800465481356e-05, 3.781871419050731e-05, 3.761126572499052e-05, 3.7542107747867703e-05, 3.7415702536236495e-05, 3.662450399133377e-05, 3.6258632462704554e-05, 3.590144115150906e-05, 3.55682241206523e-05, 3.5745335480896756e-05, 3.463883331278339e-05, 3.366340752108954e-05, 3.1926123483572155e-05, 3.0400446121348068e-05, 2.8349957574391738e-05, 2.6445855837664567e-05, 2.4638955437694676e-05, 2.3407932530972175e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.09253507107496262, 0.30484533309936523, 0.33420246839523315, 0.38682517409324646, 0.12552575767040253, 0.14749252796173096, 0.1726136952638626, 0.1892337054014206, 0.031880609691143036, 0.06123264133930206, 0.10403046011924744, 0.15778414905071259, 0.053015924990177155, 0.09510950744152069, 0.1597633957862854, 0.7120755910873413, 0.04075472429394722, 0.07218439877033234, 0.11449843645095825, 0.16539782285690308, 0.0493597537279129, 0.07179203629493713, 0.10295005142688751, 0.1323672980070114, 0.023035654798150063, 0.06294231116771698, 0.09175154566764832, 0.1252463310956955, 0.03424545377492905, 0.07851024717092514, 0.12469635903835297, 0.19299884140491486 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.125052374907682e-06, "first_to_last_ratio": 2.282830133186621, "imbalance_abs_log_ratio": 0.8254159596925686, "last_quartile_mean": 3.55920147399047e-06, "mean": 5.2053669250540224e-06, "normalized": [ 1.936708187251961, 2.057532287690085, 1.6800784059556717, 1.440095135743866, 1.5661357510227474, 1.3925489512407077, 1.246644135390895, 1.1674496789349236, 1.0945761582169642, 1.0284891213276248, 0.9743321364655805, 0.9218510024239784, 0.8793642572376682, 0.8542321734121537, 0.8405229222080287, 0.8631893221699054, 0.8729930887820545, 0.864356118998234, 0.8429990020601789, 0.8174384986744575, 0.8115121736936753, 0.7998527668126937, 0.7909597412109762, 0.7860900505247909, 0.7834756778170519, 0.7652930939089312, 0.7413802236605472, 0.7072803330050248, 0.6729535215078144, 0.6325835393606178, 0.6002196914254118, 0.5668628518647791 ], "peak_layer": 2, "peak_normalized": 2.057532287690085, "population_cv": 0.3713746971880558, "rest_mean": 5.403577578935828e-06, "spike_contrast": 0.765239201933808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1350293940922714e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.008127674140269e-05, 1.0710210517572705e-05, 8.745424565859139e-06, 7.4962235885323025e-06, 8.15231123851845e-06, 7.248728252307046e-06, 6.489240149676334e-06, 6.077003945392789e-06, 5.697670530935284e-06, 5.353663254936691e-06, 5.0717562771751545e-06, 4.798572717845673e-06, 4.577413619699655e-06, 4.446591901796637e-06, 4.375230219011428e-06, 4.493217147683026e-06, 4.544249350146856e-06, 4.499290753301466e-06, 4.388119123177603e-06, 4.255067324265838e-06, 4.224218628223753e-06, 4.163527137279743e-06, 4.117235675948905e-06, 4.0918871491157915e-06, 4.078278379893163e-06, 3.9836313590058126e-06, 3.859156095131766e-06, 3.6816536521655507e-06, 3.502970002955408e-06, 3.2928294331213692e-06, 3.1243637295119697e-06, 2.9507291401387192e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.1829198633204214e-05, "first_to_last_ratio": 2.242284345131851, "imbalance_abs_log_ratio": 0.8074951431611719, "last_quartile_mean": 1.4194987670634873e-05, "mean": 2.061655351326408e-05, "normalized": [ 1.9245211737084353, 2.037756753011099, 1.6531757560780274, 1.423897927003799, 1.5389403594243347, 1.3703405560301187, 1.238377282521067, 1.1639192115575097, 1.0938268273523595, 1.0314185318108018, 0.9796456918768245, 0.9244239585214099, 0.8827240225333328, 0.8588384283674352, 0.8455928777461329, 0.869423769586083, 0.8794014738389228, 0.8731224408861616, 0.8549747677939339, 0.8252991088856002, 0.8191238338930235, 0.807730839300627, 0.8000289282663227, 0.7953051177320747, 0.7913018779978491, 0.7709359373588888, 0.7460162927734657, 0.7109784824788987, 0.6762365500070177, 0.6361514355038691, 0.6048344862432129, 0.571735299911361 ], "peak_layer": 2, "peak_normalized": 2.037756753011099, "population_cv": 0.3620430414304777, "rest_mean": 2.136982847231492e-05, "spike_contrast": 0.7744034424903768, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6548868734389545e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.9676993765169755e-05, 4.201152114546858e-05, 3.408278644201346e-05, 2.9355867809499614e-05, 3.172764627379365e-05, 2.8251699404790998e-05, 2.5531071514706127e-05, 2.3996002710191533e-05, 2.255093932035379e-05, 2.1264295355649665e-05, 2.0196917830617167e-05, 1.905843600980006e-05, 1.8198727048002183e-05, 1.7706288417684846e-05, 1.743321081448812e-05, 1.792452167137526e-05, 1.8130227545043454e-05, 1.8000775526161306e-05, 1.762663305271417e-05, 1.7014823242789134e-05, 1.6887510355445556e-05, 1.6652626072755083e-05, 1.649383921176195e-05, 1.6396450519096106e-05, 1.631391751288902e-05, 1.5894042007857934e-05, 1.538028482173104e-05, 1.4657925930805504e-05, 1.3941667020844761e-05, 1.3115250112605281e-05, 1.2469602552300785e-05, 1.1787211406044662e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028226451249793172, "first_to_last_ratio": 2.2422842808882666, "imbalance_abs_log_ratio": 0.8074951145102182, "last_quartile_mean": 0.012588257202878594, "mean": 0.018282965727848932, "normalized": [ 1.9245212176514708, 2.0377567662716185, 1.6531757024174414, 1.4238979483152348, 1.5389402141588437, 1.3703406107590115, 1.2383773082826783, 1.1639191437690835, 1.0938267880508532, 1.0314185329225836, 0.9796457030288253, 0.9244239728212152, 0.8827239938691996, 0.8588384260470624, 0.8455928680018439, 0.869423828829971, 0.8794014231683566, 0.8731224338869547, 0.8549747774634064, 0.825299136337954, 0.8191238086543232, 0.8077308132566419, 0.8000289348300117, 0.7953051751513364, 0.7913019017816899, 0.770935989044561, 0.7460162503339971, 0.7109785284086558, 0.6762365604079628, 0.6361514610493322, 0.6048345028026112, 0.571735278225269 ], "peak_layer": 2, "peak_normalized": 2.0377567662716185, "population_cv": 0.3620430350402813, "rest_mean": 0.018950977466172643, "spike_contrast": 0.774403450539526, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.014675702340900898, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.035185955464839935, 0.037256237119436264, 0.030224954709410667, 0.026033077389001846, 0.028136391192674637, 0.02505389042198658, 0.02264120988547802, 0.021279893815517426, 0.019998397678136826, 0.01885738968849182, 0.017910828813910484, 0.016901211813092232, 0.01613881252706051, 0.01570211350917816, 0.015459945425391197, 0.015895646065473557, 0.01607806608080864, 0.015963267534971237, 0.015631474554538727, 0.015088915824890137, 0.014976012520492077, 0.014767714776098728, 0.014626901596784592, 0.014540537260472775, 0.014467345550656319, 0.014094996266067028, 0.013639389537274837, 0.012998796068131924, 0.012363609857857227, 0.011630735360085964, 0.011058168485760689, 0.010453016497194767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.3494244655021248e-06, "first_to_last_ratio": 1.5177024210142414, "imbalance_abs_log_ratio": 0.41719762616888667, "last_quartile_mean": 8.891232212704381e-07, "mean": 1.0250225503938282e-06, "normalized": [ 2.738505051354672, 1.4794936132945526, 1.3168876347326839, 1.1024576808453903, 1.0149632207688972, 0.979817680374155, 0.9622164613647413, 0.9375203415461504, 0.9109530884583005, 0.8889289387407567, 0.8999297560774143, 0.9067763813865627, 0.876468303610318, 0.8712274556920194, 0.8670923959065272, 0.8512620461270487, 0.8567121286805068, 0.8739411285350861, 0.899701222832916, 0.9374377124429466, 0.9492248371986144, 0.9687717774932662, 0.9778602024648368, 0.9925052959874211, 0.9856180215959583, 0.9661752272448075, 0.9353666611085539, 0.8925798141637848, 0.8523571867512545, 0.8001195113520716, 0.7757205790261442, 0.7314086428416404 ], "peak_layer": 1, "peak_normalized": 2.738505051354672, "population_cv": 0.34166636728519467, "rest_mean": 1.02980674312305e-06, "spike_contrast": 0.970267398546873, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.991879096560297e-07, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.8070294320059475e-06, 1.5165143167905626e-06, 1.3498395219357917e-06, 1.1300439837214071e-06, 1.040360189108469e-06, 1.004335217658081e-06, 9.862935712590115e-07, 9.609794915377279e-07, 9.337474580206617e-07, 9.111722079069295e-07, 9.224482937497669e-07, 9.294662390857411e-07, 8.983997759060003e-07, 8.930277886065596e-07, 8.887892590792035e-07, 8.725627935746161e-07, 8.781492510934186e-07, 8.958093644650944e-07, 9.222140420206415e-07, 9.608947948436253e-07, 9.7297686352249e-07, 9.9301291811571e-07, 1.0023287586591323e-06, 1.0173403097724076e-06, 1.0102806982104084e-06, 9.903513955578092e-07, 9.587719205228495e-07, 9.14914437544212e-07, 8.736853374102793e-07, 8.20140542145964e-07, 7.951310863063554e-07, 7.497103524656268e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.3858743134042015e-05, "first_to_last_ratio": 2.465036348010912, "imbalance_abs_log_ratio": 0.9022065530293095, "last_quartile_mean": 9.678860578787862e-06, "mean": 1.5052110796887064e-05, "normalized": [ 1.9687217764679332, 2.0294404332853553, 1.6968677627976259, 1.4810317330109646, 1.5821503401553387, 1.4269352026634141, 1.2874921775494743, 1.2079704664585447, 1.13377287350848, 1.0628809057173056, 1.0013232141891217, 0.9383772422695752, 0.8930109387824885, 0.8652307689960489, 0.8502741867556953, 0.8889045267143569, 0.9020099274457369, 0.8913174614788361, 0.8591545649380836, 0.8064478255585749, 0.7966230348362755, 0.7772037861237395, 0.760253544782019, 0.7484175118458054, 0.7418822737465284, 0.7197362528586176, 0.691945931996893, 0.6600449694523348, 0.6311394200398015, 0.5944298085404294, 0.5657980360452561, 0.5392111009893452 ], "peak_layer": 2, "peak_normalized": 2.0294404332853553, "population_cv": 0.3872905960483728, "rest_mean": 1.57075019160402e-05, "spike_contrast": 0.73296179207868, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1512998753460124e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.963341830763966e-05, 3.054736225749366e-05, 2.5541441573295742e-05, 2.22926537389867e-05, 2.3814702217350714e-05, 2.1478386770468205e-05, 1.937947490660008e-05, 1.818250530050136e-05, 1.7065674910554662e-05, 1.5998601156752557e-05, 1.5072027963469736e-05, 1.4124558219918981e-05, 1.3441699593386147e-05, 1.3023549399804324e-05, 1.279842126677977e-05, 1.3379889423958957e-05, 1.3577153367805295e-05, 1.3416209185379557e-05, 1.2932089703099336e-05, 1.213874202221632e-05, 1.1990858183708042e-05, 1.1698557500494644e-05, 1.1443420589785092e-05, 1.12652633106336e-05, 1.1166894182679243e-05, 1.0833549822564237e-05, 1.0415246833872516e-05, 9.93507001112448e-06, 9.499980478722136e-06, 8.947423339122906e-06, 8.516454727214295e-06, 8.116265235003084e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7515224044473143e-05, "first_to_last_ratio": 2.359437553652934, "imbalance_abs_log_ratio": 0.8584232659103358, "last_quartile_mean": 7.423474301049282e-06, "mean": 1.1241153700325413e-05, "normalized": [ 1.9309435555261214, 1.9526313215933524, 1.6582244442374867, 1.4588157040411531, 1.5412217068718899, 1.4258707789492027, 1.2881056965786792, 1.2092589806313907, 1.137794759023235, 1.0622568400477936, 0.9971593988413544, 0.9361807205321625, 0.8867849352218846, 0.8639024876188973, 0.8515161822097123, 0.9048685915984879, 0.9265194638027697, 0.9115225939595661, 0.8653717695293225, 0.8111735521509221, 0.8039114483119388, 0.7800093222994203, 0.76323256629171, 0.7496538447594057, 0.745119944996388, 0.7253817283186728, 0.6970650412445614, 0.6707070461181087, 0.6492809756642738, 0.6191651059644422, 0.5991054029612273, 0.5772440901044666 ], "peak_layer": 2, "peak_normalized": 1.9526313215933524, "population_cv": 0.3678078470590104, "rest_mean": 1.1723304632377457e-05, "spike_contrast": 0.7367836065087988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.63753866724437e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.170603329432197e-05, 2.1949828806100413e-05, 1.8640355847310275e-05, 1.639877154957503e-05, 1.7325110093224794e-05, 1.602843258297071e-05, 1.4479794117505662e-05, 1.3593466064776294e-05, 1.2790125765604898e-05, 1.1940992408199236e-05, 1.1209222066099755e-05, 1.052375137078343e-05, 9.96848575596232e-06, 9.711260645417497e-06, 9.572024282533675e-06, 1.0171766916755587e-05, 1.0415147698950022e-05, 1.0246565580018796e-05, 9.727777069201693e-06, 9.118526577367447e-06, 9.036892151925713e-06, 8.768204679654445e-06, 8.579614586778916e-06, 8.426974090980366e-06, 8.376007826882415e-06, 8.154127499437891e-06, 7.835815267753787e-06, 7.5395209933049046e-06, 7.298667242139345e-06, 6.960130122024566e-06, 6.734635917382548e-06, 6.488889539468801e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.572684515049332e-05, "first_to_last_ratio": 2.139075374807702, "imbalance_abs_log_ratio": 0.7603736678186844, "last_quartile_mean": 3.072675508519751e-05, "mean": 4.3482417027007614e-05, "normalized": [ 1.9051243467602497, 1.9135350736361767, 1.6339952020703241, 1.4129520831778495, 1.4905720952473844, 1.3455550767012106, 1.2384699946115487, 1.15237968581952, 1.1017299217830712, 1.0257825506862357, 0.9642904052505235, 0.9205031377378704, 0.884021208319066, 0.8684631875068248, 0.8525398830567881, 0.8631133634913525, 0.8692805159213056, 0.8642712942167508, 0.865837261658028, 0.84731405133431, 0.8430444327809488, 0.8356094968046114, 0.8292110092261231, 0.819222183409267, 0.8216396147950802, 0.7972044318616739, 0.7748498429513314, 0.7322001754439701, 0.700092530137593, 0.6526455238420115, 0.6087378616535022, 0.5658125581074975 ], "peak_layer": 2, "peak_normalized": 1.9135350736361767, "population_cv": 0.3397683108879149, "rest_mean": 4.4853358472378374e-05, "spike_contrast": 0.8043842078007138, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6079333222005515e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.283941133413464e-05, 8.320513006765395e-05, 7.10500607965514e-05, 6.14385717199184e-05, 6.481367745436728e-05, 5.850798697792925e-05, 5.385166878113523e-05, 5.0108254072256386e-05, 4.790587991010398e-05, 4.4603504647966474e-05, 4.1929677536245435e-05, 4.002570130978711e-05, 3.8439378840848804e-05, 3.7762878491776064e-05, 3.707049472723156e-05, 3.75302552129142e-05, 3.779841790674254e-05, 3.758060483960435e-05, 3.7648696888936684e-05, 3.684326293296181e-05, 3.66576095984783e-05, 3.63343206117861e-05, 3.6056098906556144e-05, 3.5621760616777465e-05, 3.572687637642957e-05, 3.466437556198798e-05, 3.3692344004521146e-05, 3.1837833375902846e-05, 3.0441715352935717e-05, 2.837860483850818e-05, 2.646939356054645e-05, 2.4602897610748187e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.15345558524131775, 0.3249167501926422, 0.3818626403808594, 0.40600064396858215, 0.1351035088300705, 0.15324510633945465, 0.18001793324947357, 0.20391276478767395, 0.04374876990914345, 0.08211024105548859, 0.1380448192358017, 0.2295975536108017, 0.08199761807918549, 0.13892096281051636, 0.21687114238739014, 0.7360114455223083, 0.057135771960020065, 0.08251863718032837, 0.12661415338516235, 0.19619159400463104, 0.06753352284431458, 0.095451220870018, 0.1274663656949997, 0.15628865361213684, 0.058436937630176544, 0.08794140070676804, 0.12247690558433533, 0.1640285700559616, 0.07537907361984253, 0.12290703505277634, 0.19157102704048157, 0.29249706864356995 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.9018017800081e-06, "first_to_last_ratio": 2.3198553421908015, "imbalance_abs_log_ratio": 0.8415048312302555, "last_quartile_mean": 3.4061614258007467e-06, "mean": 5.078743377850969e-06, "normalized": [ 1.847125037232453, 2.083137317368821, 1.641111481706062, 1.4465372937027328, 1.6030646026913287, 1.4179987767162792, 1.2455258536342846, 1.1623609049834964, 1.0875552599581997, 1.028118506110977, 0.9635771142568214, 0.9122601462896323, 0.8813638792944334, 0.8604039723922948, 0.8561688509558816, 0.8887307288299607, 0.8917013756490454, 0.8870568803314235, 0.8597145194427529, 0.8329319585333728, 0.8191043976245732, 0.810440302071778, 0.8055455442874069, 0.803104343609964, 0.7839286872176334, 0.7606233867362351, 0.7253002050295685, 0.689251306939001, 0.6485362450095404, 0.6152354884679782, 0.5812522182760415, 0.5612334146500276 ], "peak_layer": 2, "peak_normalized": 2.083137317368821, "population_cv": 0.36840679032180734, "rest_mean": 5.262683560903059e-06, "spike_contrast": 0.7763085775707618, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.085466389369685e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 9.381074050907046e-06, 1.0579719855741132e-05, 8.334784070029855e-06, 7.346591701207217e-06, 8.14155373518588e-06, 7.201651897048578e-06, 6.325706181087298e-06, 5.9033327488577925e-06, 5.523414074559696e-06, 5.221550054557156e-06, 4.893760888080578e-06, 4.633135176845826e-06, 4.476220965443645e-06, 4.369770977064036e-06, 4.348261882114457e-06, 4.5136353037378285e-06, 4.528722456598189e-06, 4.505134256760357e-06, 4.36626942246221e-06, 4.230247668601805e-06, 4.160021035204409e-06, 4.116018317290582e-06, 4.091159098607022e-06, 4.078760866832454e-06, 3.9813726289139595e-06, 3.863010988425231e-06, 3.6836136132478714e-06, 3.5005305107915774e-06, 3.293749159638537e-06, 3.1246231628756505e-06, 2.9520308544306317e-06, 2.850360488082515e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.093152440669655e-05, "first_to_last_ratio": 2.2774685844866087, "imbalance_abs_log_ratio": 0.8230645561980515, "last_quartile_mean": 1.3581537245954678e-05, "mean": 2.01113496416383e-05, "normalized": [ 1.8305028390484899, 2.063895445405066, 1.6144389462797186, 1.4292254312672816, 1.5742789705777873, 1.395222470726611, 1.2383859090556808, 1.15815685579556, 1.0870603073810894, 1.0303295548521039, 0.9686496018996943, 0.9150613022126579, 0.8858369611012855, 0.866010221809998, 0.8620422691094124, 0.8950413704487017, 0.898565866832108, 0.8961986259205167, 0.8726168436106015, 0.8416436403028231, 0.8273619591175169, 0.8203659676564082, 0.8150936944275661, 0.8114785712381606, 0.790107287070579, 0.7656015997474095, 0.7288914121037795, 0.6928588452491169, 0.6522528807444096, 0.6198966689630461, 0.586304513764539, 0.566623166280281 ], "peak_layer": 2, "peak_normalized": 2.063895445405066, "population_cv": 0.3585699713300157, "rest_mean": 2.0808239578737238e-05, "spike_contrast": 0.7856572354159773, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.634814398130402e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.6813882616115734e-05, 4.1507722926326096e-05, 3.246854612370953e-05, 2.874365236493759e-05, 3.166087481076829e-05, 2.805980693665333e-05, 2.4905612008296885e-05, 2.329209746676497e-05, 2.186224992328789e-05, 2.072131792374421e-05, 1.9480850824038498e-05, 1.8403117792331614e-05, 1.7815376850194298e-05, 1.7416634364053607e-05, 1.7336833479930647e-05, 1.800048994482495e-05, 1.8071372323902324e-05, 1.8023763914243318e-05, 1.7549502445035614e-05, 1.6926589523791336e-05, 1.6639365640003234e-05, 1.6498666809638962e-05, 1.6392634279327467e-05, 1.631992927286774e-05, 1.5890123904682696e-05, 1.539728145871777e-05, 1.4658990039606579e-05, 1.3934326489106752e-05, 1.3117685739416629e-05, 1.246695865120273e-05, 1.1791375072789378e-05, 1.1395556612114888e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027430385118350387, "first_to_last_ratio": 2.2774685803861345, "imbalance_abs_log_ratio": 0.8230645543975988, "last_quartile_mean": 0.012044243048876524, "mean": 0.017834946047514677, "normalized": [ 1.8305029039623888, 2.0638953970410903, 1.6144389639181254, 1.4292254712142256, 1.5742789179863164, 1.3952224600071201, 1.2383859063079345, 1.158156779896113, 1.0870603341935974, 1.0303295369704457, 0.9686496343170561, 0.9150613749177388, 0.8858370264114429, 0.866010161028954, 0.8620423023293187, 0.8950413515523591, 0.8985659233861498, 0.8961986284425522, 0.8726168533342663, 0.8416436914062322, 0.827361906598272, 0.82036592184569, 0.8150936331886041, 0.8114785658737781, 0.790107272131739, 0.7656015867257281, 0.7288914394956378, 0.6928588334464092, 0.6522528897000817, 0.6198967082595142, 0.5863045037167507, 0.5666231203943681 ], "peak_layer": 2, "peak_normalized": 2.0638953970410903, "population_cv": 0.35856997121771, "rest_mean": 0.018452955133937025, "spike_contrast": 0.7856571955876671, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.014497696980834007, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.032646920531988144, 0.03680946305394173, 0.02879343181848526, 0.025490159168839455, 0.028077179566025734, 0.02488371729850769, 0.022086545825004578, 0.020655663684010506, 0.019387662410736084, 0.01837587170302868, 0.017275813966989517, 0.01632007025182247, 0.01579885557293892, 0.015445244498550892, 0.015374477952718735, 0.015963014215230942, 0.016025874763727188, 0.01598365418612957, 0.015563074499368668, 0.015010669827461243, 0.014755954965949059, 0.014631181955337524, 0.014537150971591473, 0.014472676441073418, 0.014091520570218563, 0.013654462993144989, 0.01299973949790001, 0.012357099913060665, 0.011632895097136497, 0.011055824346840382, 0.010456709191203117, 0.010105692781507969 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2912549820498498e-06, "first_to_last_ratio": 1.5202665797334127, "imbalance_abs_log_ratio": 0.4188857008842256, "last_quartile_mean": 8.49360894505935e-07, "mean": 9.93022506889929e-07, "normalized": [ 2.660532038406061, 1.475963218576449, 1.19801710692682, 1.0899716081643689, 1.0508156804583857, 1.0174876453067656, 0.9698761501636428, 0.9399606463412474, 0.9117614543001846, 0.9036711533632844, 0.9003299466006321, 0.9006091198840781, 0.8847274400701823, 0.8793430077222689, 0.8796797672930117, 0.8766142420011561, 0.8797972295808179, 0.8997445814178318, 0.9212125890421617, 0.9587869545893736, 0.9600087455596155, 0.9836840367208892, 1.0020615039365162, 1.0127125628875548, 0.9873575381135197, 0.9639449915433914, 0.9154718200755378, 0.8777908111354492, 0.8263667145220306, 0.7874398153801374, 0.7530554210378857, 0.7312044588787489 ], "peak_layer": 1, "peak_normalized": 2.660532038406061, "population_cv": 0.3259399130744097, "rest_mean": 9.950150106979656e-07, "spike_contrast": 0.9871840884465768, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.82262986326532e-07, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.6419681944389595e-06, 1.4656646953881136e-06, 1.189657950817491e-06, 1.082366338778229e-06, 1.0434836212880327e-06, 1.0103881322720554e-06, 9.631088460082537e-07, 9.334020774076635e-07, 9.053996450347768e-07, 8.973657941169222e-07, 8.940479006014357e-07, 8.943251259552198e-07, 8.785542604528018e-07, 8.732073979444976e-07, 8.735418077776558e-07, 8.704976721674029e-07, 8.736584504731582e-07, 8.934666198001651e-07, 9.147848345492093e-07, 9.520970252197003e-07, 9.533102911518654e-07, 9.768203881321824e-07, 9.950696266969317e-07, 1.0056463679575245e-06, 9.80468257694156e-07, 9.5721907200641e-07, 9.090841217584966e-07, 8.71666031798668e-07, 8.206007464650611e-07, 7.81945459493727e-07, 7.478009820260922e-07, 7.261024848048692e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.32548172789393e-05, "first_to_last_ratio": 2.4975346081461725, "imbalance_abs_log_ratio": 0.9153040885601471, "last_quartile_mean": 9.311109124610084e-06, "mean": 1.4726773656548175e-05, "normalized": [ 1.8853895701442087, 2.0473234930293196, 1.6582347285378365, 1.4857598427569607, 1.6139858065355936, 1.4508395759575576, 1.287840205679765, 1.2033012904754463, 1.1275415086272205, 1.0620110592815606, 0.9899911509412549, 0.9287416361171535, 0.8955665859807346, 0.8710690774451937, 0.8664988688923737, 0.9137534246592263, 0.9194604727439549, 0.9140176249806901, 0.8758139373559145, 0.8223144839018022, 0.8027749000523561, 0.7865160814383286, 0.7720189070902561, 0.7611779242876019, 0.7378071887453072, 0.7143310324576317, 0.6774044365690762, 0.6461920526823346, 0.6100565755684288, 0.5819809992384402, 0.553681426983732, 0.5366041308427388 ], "peak_layer": 2, "peak_normalized": 2.0473234930293196, "population_cv": 0.3835372248130478, "rest_mean": 1.534840986197297e-05, "spike_contrast": 0.7407893227704523, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1369938147254288e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.7765705453930423e-05, 3.0150469683576375e-05, 2.4420447516604327e-05, 2.1880448912270367e-05, 2.376880365773104e-05, 2.1366186047089286e-05, 1.8965731214848347e-05, 1.7720745745464228e-05, 1.6605048585915938e-05, 1.563999649079051e-05, 1.4579375601897482e-05, 1.3677367860509548e-05, 1.3188806406105869e-05, 1.2828037142753601e-05, 1.2760732715833e-05, 1.3456639862852171e-05, 1.3540686268243007e-05, 1.3460530681186356e-05, 1.2897913620690815e-05, 1.2110039278923068e-05, 1.1822284250229131e-05, 1.1582844308577478e-05, 1.1369347703293897e-05, 1.1209695003344677e-05, 1.0865519470826257e-05, 1.051979143085191e-05, 9.975981811294332e-06, 9.516324098512996e-06, 8.984165106085129e-06, 8.570702448196243e-06, 8.153941053024027e-06, 7.902447578089777e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7115447462856537e-05, "first_to_last_ratio": 2.384273554189065, "imbalance_abs_log_ratio": 0.8688944885049729, "last_quartile_mean": 7.178474731972528e-06, "mean": 1.1022389131198906e-05, "normalized": [ 1.8533266693628003, 1.9634876569720725, 1.62746111449772, 1.4631152334273978, 1.5665541851429352, 1.44292255643436, 1.297243679761451, 1.2082029846844278, 1.1325776467348811, 1.0586573654925269, 0.9865546892223965, 0.9267660638529281, 0.890239943667283, 0.8665873151718257, 0.8665599207260849, 0.9296718554228318, 0.9425328050776204, 0.9351613887984032, 0.8819628604108036, 0.823795705683662, 0.8070268393278794, 0.7865168158338803, 0.7700336933503881, 0.7629366340955328, 0.7426802567648084, 0.7143746981629545, 0.6833977272777232, 0.6635906353598262, 0.6326960953519897, 0.6128141637886501, 0.5878658687743828, 0.5726849313675726 ], "peak_layer": 2, "peak_normalized": 1.9634876569720725, "population_cv": 0.363814433471477, "rest_mean": 1.1484025469178084e-05, "spike_contrast": 0.7427319739932458, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.529552906111348e-06, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.04280877369456e-05, 2.164232500945218e-05, 1.7938509699888527e-05, 1.61270254466217e-05, 1.7267169823753648e-05, 1.590445390320383e-05, 1.4298724636319093e-05, 1.3317283446667716e-05, 1.2483711543609388e-05, 1.1668933439068496e-05, 1.0874189683818258e-05, 1.0215176189376507e-05, 9.812571079237387e-06, 9.551862603984773e-06, 9.551560651743785e-06, 1.0247204954794142e-05, 1.038896334648598e-05, 1.0307712727808394e-05, 9.72133784671314e-06, 9.080196832655929e-06, 8.895363862393424e-06, 8.669294402352534e-06, 8.487611012242269e-06, 8.409384463448077e-06, 8.186110790120438e-06, 7.874115908634849e-06, 7.532675681432011e-06, 7.3143542067555245e-06, 6.973822564759757e-06, 6.754676178388763e-06, 6.47968636258156e-06, 6.312356163107324e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.367657533701276e-05, "first_to_last_ratio": 2.18189651420632, "imbalance_abs_log_ratio": 0.7801944592496646, "last_quartile_mean": 2.918404925367213e-05, "mean": 4.231288824030344e-05, "normalized": [ 1.7779407300288732, 1.9563222737204058, 1.603774689185993, 1.4143174555706712, 1.5327227884698011, 1.3724537206523966, 1.2293267806538501, 1.152324512384129, 1.0911327441927006, 1.0293532301829285, 0.9557045350033907, 0.9145594044662916, 0.8946717401908558, 0.8760144215357385, 0.8707713092633232, 0.8938212455104791, 0.8890009731933908, 0.8873877674020535, 0.8842233459795642, 0.8657151992078328, 0.8569039986584633, 0.8485896651489135, 0.8407244808721062, 0.844482236684939, 0.8186411956248972, 0.7956332166567571, 0.7546037265710437, 0.7184850006798041, 0.6700028489089177, 0.624855570006522, 0.582328729525934, 0.5532104638670327 ], "peak_layer": 2, "peak_normalized": 1.9563222737204058, "population_cv": 0.33607795885441427, "rest_mean": 4.3551963851011046e-05, "spike_contrast": 0.8179167319375746, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.562187994248234e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.522980740759522e-05, 8.277764572994784e-05, 6.78603391861543e-05, 5.9843856433872133e-05, 6.485392805188894e-05, 5.807248089695349e-05, 5.2016366680618376e-05, 4.875817830907181e-05, 4.616897786036134e-05, 4.355490818852559e-05, 4.043861918034963e-05, 3.8697649870300665e-05, 3.7856145354453474e-05, 3.7066700315335765e-05, 3.68448490917217e-05, 3.782015846809372e-05, 3.761619882425293e-05, 3.754793942789547e-05, 3.741404361790046e-05, 3.6630910472013056e-05, 3.625808312790468e-05, 3.5906279663322493e-05, 3.557348100002855e-05, 3.57324825017713e-05, 3.463907341938466e-05, 3.3665539376670495e-05, 3.1929463148117065e-05, 3.040117553609889e-05, 2.8349755666567944e-05, 2.6439443900017068e-05, 2.4640010451548733e-05, 2.3407932530972175e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.06279198825359344, 0.07690861076116562, 0.18058517575263977, 0.15882407128810883, 0.19892723858356476, 0.15647411346435547, 0.1533493995666504, 0.1527898609638214, 0.16355258226394653, 0.11789151281118393, 0.11456070095300674, 0.12148918211460114, 0.15094918012619019, 0.09854138642549515, 0.11213528364896774, 0.16254977881908417, 0.2323070764541626, 0.1390199512243271, 0.14812250435352325, 0.15333011746406555, 0.1781008094549179, 0.07593096792697906, 0.09691466391086578, 0.09368696808815002, 0.10724716633558273, 0.06538715958595276, 0.07412076741456985, 0.07655765861272812, 0.08420819044113159, 0.07230556756258011, 0.07723049819469452, 0.08422473073005676 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.718497104429844e-06, "first_to_last_ratio": 5.4600950128558265, "imbalance_abs_log_ratio": 1.6974661912273534, "last_quartile_mean": 1.2304725629519453e-06, "mean": 3.460733235982616e-06, "normalized": [ 4.1554392029645175, 2.1310249348511077, 2.186628621017466, 1.156264880926304, 1.051728395133589, 1.9487043640861321, 1.6569575754204475, 1.2440581208520776, 0.7492436301896301, 1.1025523326858364, 1.1770939413199029, 1.0278305718772012, 0.5042914345692446, 0.5528163951643698, 0.4411680047894231, 0.9973694803382674, 0.4275370158551335, 0.7739052934101094, 1.5627780657381267, 1.739415630377695, 0.5961441757664185, 0.7016049433942307, 0.7230338540262741, 0.5479886519610823, 0.2846566311183848, 0.3910340264134103, 0.5107042475345903, 0.37580476181367506, 0.29294599638498453, 0.45911022842212285, 0.29896279650123675, 0.231201795097008 ], "peak_layer": 1, "peak_normalized": 4.1554392029645175, "population_cv": 0.7923058521157783, "rest_mean": 3.7358707240181737e-06, "spike_contrast": 0.5286560875603242, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9749908005906037e-06, "top_five_layers": [ 1, 3, 2, 6, 20 ], "top_five_spike_overlap": 0 }, "values": [ 1.4380866559804417e-05, 7.374908818746917e-06, 7.5673383435059804e-06, 4.001524303021142e-06, 3.6397514122654684e-06, 6.743945959897246e-06, 5.7342881518707145e-06, 4.305353286326863e-06, 2.592932332845521e-06, 3.815639502136037e-06, 4.073608124599559e-06, 3.557047421054449e-06, 1.7452181282351376e-06, 1.913150072141434e-06, 1.5267647768268944e-06, 3.4516297091613524e-06, 1.479591560382687e-06, 2.6782797704072436e-06, 5.408357992564561e-06, 6.019653483235743e-06, 2.063095962512307e-06, 2.428067546134116e-06, 2.50222728936933e-06, 1.8964425407830277e-06, 9.851206641542376e-07, 1.3532644516089931e-06, 1.7674111632004497e-06, 1.300560029449116e-06, 1.0138079460375593e-06, 1.5888580264800112e-06, 1.0346304861741373e-06, 8.001277365110582e-07 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.6456544674147153e-05, "first_to_last_ratio": 5.448763030808658, "imbalance_abs_log_ratio": 1.6953886160901857, "last_quartile_mean": 4.855513907386921e-06, "mean": 1.3720428839292254e-05, "normalized": [ 4.173598008637582, 2.103709111147759, 2.234357653663474, 1.1576980683143674, 1.0174050734502822, 1.9214205167195906, 1.5972559788378484, 1.2206300853156902, 0.7360677134431396, 1.1059605072991032, 1.1320345474510969, 1.045897352609526, 0.49038680071526464, 0.5622676476822898, 0.4266762355069258, 1.004384658425444, 0.4110778614318569, 0.7749237983047368, 1.5517430250838387, 1.9055106902179004, 0.6175511328557414, 0.6917605316674716, 0.7266963661277974, 0.5598716813185096, 0.2878797501636235, 0.3879341985116693, 0.5111757613209584, 0.3605598270780299, 0.2888857643708419, 0.4593768179608925, 0.3088935915519739, 0.2264092428147734 ], "peak_layer": 1, "peak_normalized": 4.173598008637582, "population_cv": 0.7986066807104257, "rest_mean": 1.4795826161759948e-05, "spike_contrast": 0.5348321351881458, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.9132832979667e-06, "top_five_layers": [ 1, 3, 2, 6, 20 ], "top_five_spike_overlap": 0 }, "values": [ 5.726355448132381e-05, 2.8863791158073582e-05, 3.0656345188617706e-05, 1.588411396369338e-05, 1.3959233911009505e-05, 2.6362713470007293e-05, 2.191503699577879e-05, 1.674756822467316e-05, 1.009916468319716e-05, 1.5174252439464908e-05, 1.5531999451923184e-05, 1.435016019968316e-05, 6.72831720294198e-06, 7.714553248661105e-06, 5.854180926689878e-06, 1.3780588233203162e-05, 5.640164545184234e-06, 1.0632286830514204e-05, 2.1290579752530903e-05, 2.614442382764537e-05, 8.473066372971516e-06, 9.49125114857452e-06, 9.970585779228713e-06, 7.681679562665522e-06, 3.949833626393229e-06, 5.322623565007234e-06, 7.0135506575752515e-06, 4.947035449731629e-06, 3.963636572734686e-06, 6.3028469412529375e-06, 4.238152541802265e-06, 3.1064319045981392e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.023461928823962808, "first_to_last_ratio": 5.448763126593669, "imbalance_abs_log_ratio": 1.6953886336694068, "last_quartile_mean": 0.004305918293539435, "mean": 0.012167413689894602, "normalized": [ 4.173598084973268, 2.1037090871122417, 2.234357712750731, 1.157698009629034, 1.0174050464680184, 1.9214205450906374, 1.59725599079182, 1.2206300674885242, 0.7360677803275659, 1.1059605017352738, 1.1320345805663141, 1.0458974046616514, 0.4903867781433125, 0.5622676330882879, 0.42667620792714267, 1.0043846550160789, 0.41107786974695965, 0.7749237446127663, 1.551743137345238, 1.9055106029576432, 0.6175510806823439, 0.691760545613661, 0.7266963928653994, 0.5598716275528304, 0.2878797436072962, 0.3879341769467224, 0.5111757600120082, 0.3605598347532703, 0.28888575899731955, 0.4593768268733131, 0.30889356866228096, 0.226409243001046 ], "peak_layer": 1, "peak_normalized": 4.173598084973268, "population_cv": 0.7986066956152146, "rest_mean": 0.013121086836758035, "spike_contrast": 0.534832120550615, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.007017578696832061, "top_five_layers": [ 1, 3, 2, 6, 20 ], "top_five_spike_overlap": 0 }, "values": [ 0.050781894475221634, 0.025596698746085167, 0.027186354622244835, 0.014086190611124039, 0.01237918809056282, 0.02337871864438057, 0.019434474408626556, 0.01485191099345684, 0.008956041187047958, 0.013456678949296474, 0.013773933053016663, 0.01272586639970541, 0.005966738797724247, 0.006841342896223068, 0.005191545933485031, 0.012220763601362705, 0.005001754499971867, 0.00942881777882576, 0.018880700692534447, 0.023185135796666145, 0.0075139994733035564, 0.008416936732828617, 0.008842015638947487, 0.00681218970566988, 0.0035027519334107637, 0.004720155615359545, 0.0062196869403123856, 0.004387080669403076, 0.0035149925388395786, 0.005589427892118692, 0.0037584358360618353, 0.002754814922809601 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.9513348138721085e-06, "first_to_last_ratio": 3.6583742473448075, "imbalance_abs_log_ratio": 1.297018854015045, "last_quartile_mean": 8.067339791750783e-07, "mean": 1.2634741715089604e-06, "normalized": [ 12.309526402394347, 3.186602737808338, 0.716414074901559, 0.5881410276372666, 0.41150136269951676, 0.4806569456453473, 0.5688709171733375, 0.42539470014768993, 0.37578688206301336, 0.4971639190900172, 0.5295259641376998, 0.5986589622164143, 0.3870590927171303, 0.5372409907363719, 0.5606252361769879, 0.5642027331159046, 0.2035736261311389, 0.4495004466120918, 0.6101764796194364, 0.688524328430386, 0.29283475717020535, 0.6520152158906203, 0.6017552935102983, 0.656211682155798, 0.4004572277104296, 0.6993974572841979, 0.7821063899475319, 0.6457741443048106, 0.6234421647147482, 0.8431613839218123, 0.5292562954255609, 0.5844411585099919 ], "peak_layer": 1, "peak_normalized": 12.309526402394347, "population_cv": 2.086670622186628, "rest_mean": 1.375629770555625e-06, "spike_contrast": 0.47820565586572594, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.578339366569708e-07, "top_five_layers": [ 1, 2, 30, 27, 3 ], "top_five_spike_overlap": 0 }, "values": [ 1.555276867293287e-05, 4.026190254080575e-06, 9.051706797436054e-07, 7.431009976244241e-07, 5.199213433115801e-07, 6.072976361792826e-07, 7.18753710771125e-07, 5.374752163334051e-07, 4.7479701947850117e-07, 6.281537707764073e-07, 6.690423788313637e-07, 7.563901363027981e-07, 4.89039166495786e-07, 6.787901156712906e-07, 7.08335505805735e-07, 7.128555807867087e-07, 2.5721001861711557e-07, 5.679322043761204e-07, 7.709422220614215e-07, 8.699327054273454e-07, 3.699891522046528e-07, 8.238043847086374e-07, 7.603022709190554e-07, 8.291065114462981e-07, 5.059673640062101e-07, 8.836706228976254e-07, 9.881712230708217e-07, 8.159189519574284e-07, 7.877030725467193e-07, 1.0653126309989602e-06, 6.687016593787121e-07, 7.384263085441489e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.5771814105391968e-05, "first_to_last_ratio": 4.993572550529696, "imbalance_abs_log_ratio": 1.6081515955891383, "last_quartile_mean": 3.1584229418513132e-06, "mean": 8.800217244697706e-06, "normalized": [ 2.6265759069342214, 2.1186852630402058, 2.2538421407700397, 1.2139819550758302, 1.0751886352347675, 2.0201548997891736, 1.7460426507992353, 1.2831874191256807, 0.8200277209467072, 1.2322326860980624, 1.2530395508876175, 1.027820633392001, 0.5423322546419725, 0.6355820979921598, 0.4727811911372265, 1.062578905841453, 0.460994689153592, 0.8637231601610539, 1.753316568512886, 1.8155319115306476, 0.5982235617014492, 0.804229051128676, 0.8286878603973532, 0.62001658378724, 0.31698601103139457, 0.41847205574491303, 0.5364769069396538, 0.33379863905874946, 0.2942071158594462, 0.4522629804378397, 0.2919398687178099, 0.2270791241309413 ], "peak_layer": 1, "peak_normalized": 2.6265759069342214, "population_cv": 0.6434515693524889, "rest_mean": 9.397281650712506e-06, "spike_contrast": 0.5933704723849582, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.57606945221778e-06, "top_five_layers": [ 1, 3, 2, 6, 20 ], "top_five_spike_overlap": 0 }, "values": [ 2.311443859071005e-05, 1.8644890587893315e-05, 1.9834300474030897e-05, 1.0683304935810156e-05, 9.461893569095992e-06, 1.777780198608525e-05, 1.5365554645541124e-05, 1.1292328053968959e-05, 7.21642209100537e-06, 1.0843915333680343e-05, 1.102702026400948e-05, 9.045044862432405e-06, 4.772641659656074e-06, 5.593260539171752e-06, 4.160577191214543e-06, 9.350925211037975e-06, 4.056853413203498e-06, 7.600951448694104e-06, 1.5429566701641306e-05, 1.5977075236150995e-05, 5.264497303869575e-06, 7.077390364429448e-06, 7.2926331995404325e-06, 5.456280632643029e-06, 2.7895457606064156e-06, 3.6826450013904832e-06, 4.721113327832427e-06, 2.937500539701432e-06, 2.5890865344990743e-06, 3.980012479587458e-06, 2.5691342671052553e-06, 1.9983456240879605e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.604969477550185e-06, "first_to_last_ratio": 4.6554299483226265, "imbalance_abs_log_ratio": 1.538034269154005, "last_quartile_mean": 2.063175600142131e-06, "mean": 5.666794919534368e-06, "normalized": [ 1.5974352749631073, 2.1598435301941388, 2.226941855679588, 1.2194393211432404, 1.1026312268209173, 2.0507201176776584, 1.8513987535553562, 1.3512399358016263, 0.8771842556929774, 1.317420097511909, 1.3400408855092742, 0.9362708096810504, 0.5836862581919815, 0.7178415815889991, 0.5251745185955801, 1.0976134976177288, 0.4952393518791189, 0.9200124618684795, 1.9599358762567376, 1.6538913088117144, 0.5562244464626633, 0.9341508255125205, 0.947277666692926, 0.665733569627415, 0.34870485527422174, 0.4467667219601853, 0.5556686106558906, 0.325474706243329, 0.2993237409593487, 0.43817195080407734, 0.2727026444295764, 0.22583934233666234 ], "peak_layer": 3, "peak_normalized": 2.226941855679588, "population_cv": 0.5912253593182908, "rest_mean": 5.991671986083492e-06, "spike_contrast": 0.6529828016714437, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.912458760169102e-06, "top_five_layers": [ 3, 2, 6, 19, 7 ], "top_five_spike_overlap": 0 }, "values": [ 9.052338100445922e-06, 1.223939034389332e-05, 1.2619622793863527e-05, 6.910312549734954e-06, 6.248385034268722e-06, 1.1621010344242677e-05, 1.0491497050679754e-05, 7.657199603272602e-06, 4.9708232836565e-06, 7.465549515472958e-06, 7.5937368819722906e-06, 5.305654667608906e-06, 3.3076303225243464e-06, 4.067861027579056e-06, 2.976056293846341e-06, 6.219950591912493e-06, 2.806419843182084e-06, 5.213521944824606e-06, 1.110655466618482e-05, 9.37226286623627e-06, 3.152009867335437e-06, 5.293641152093187e-06, 5.368028269003844e-06, 3.772575610128115e-06, 1.9760389022849267e-06, 2.5317353902210016e-06, 3.1488600598095218e-06, 1.8443984117766377e-06, 1.6962062545644585e-06, 2.4830305846990086e-06, 1.545349959997111e-06, 1.279785237784381e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.9967670520345564e-05, "first_to_last_ratio": 5.362751833258153, "imbalance_abs_log_ratio": 1.6794772450644015, "last_quartile_mean": 9.317542946973845e-06, "mean": 2.6670801545947143e-05, "normalized": [ 3.476508751938897, 2.0142464518918617, 2.234943048215937, 1.2211413922997942, 1.0839213513178267, 2.0139857856539773, 1.6922839548152497, 1.2509464146134932, 0.7654291525620647, 1.1423734022107805, 1.2080765712998793, 1.1341630295311584, 0.5240637082381794, 0.5667171140044547, 0.4265924381702442, 1.0426236214132871, 0.4450052137430249, 0.8079063596821926, 1.5538794579061908, 1.911286561600711, 0.633864177631036, 0.7039911705961566, 0.7649962092658604, 0.5862251379064324, 0.29627943867847506, 0.39112771210174047, 0.5128146165033627, 0.3335988370842466, 0.2814936032846977, 0.4583551171276488, 0.3049685860807425, 0.21619161263039666 ], "peak_layer": 1, "peak_normalized": 3.476508751938897, "population_cv": 0.7197953943382049, "rest_mean": 2.86608817953459e-05, "spike_contrast": 0.5556133378206015, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.592436819919385e-05, "top_five_layers": [ 1, 3, 2, 6, 20 ], "top_five_spike_overlap": 0 }, "values": [ 9.27212749957107e-05, 5.372156738303602e-05, 5.9607722505461425e-05, 3.25688197335694e-05, 2.890905125241261e-05, 5.3714615205535665e-05, 4.513456951826811e-05, 3.336374356877059e-05, 2.0414609025465325e-05, 3.0468014301732183e-05, 3.222037048544735e-05, 3.0249037081375718e-05, 1.3977199159853626e-05, 1.5114799680304714e-05, 1.137756225944031e-05, 2.7807607693830505e-05, 1.1868645742652006e-05, 2.154751018679235e-05, 4.144321064813994e-05, 5.0975544581888244e-05, 1.6905665688682348e-05, 1.877600880106911e-05, 2.0403062080731615e-05, 1.5635094314347953e-05, 7.902010111138225e-06, 1.0431689588585868e-05, 1.3677176866622176e-05, 8.897348379832692e-06, 7.507660029659746e-06, 1.222469836648088e-05, 8.133756637107581e-06, 5.766003596363589e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0596221424639225, 0.07634982466697693, 0.06968431174755096, 0.11288930475711823, 0.11885104328393936, 0.1495160460472107, 0.11554427444934845, 0.13194940984249115, 0.08884616196155548, 0.10897023230791092, 0.09843935817480087, 0.09431816637516022, 0.09025302529335022, 0.0895693451166153, 0.10807885229587555, 0.2204425185918808, 0.15380074083805084, 0.08548793941736221, 0.09071414917707443, 0.10860678553581238, 0.04808543995022774, 0.05631518363952637, 0.07238366454839706, 0.08572373539209366, 0.06546958535909653, 0.06323372572660446, 0.06869969516992569, 0.07729499042034149, 0.04753003641963005, 0.06484207510948181, 0.07835857570171356, 0.08203105628490448 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.5725815504529237e-06, "first_to_last_ratio": 0.39907105360304757, "imbalance_abs_log_ratio": 0.9186157987408602, "last_quartile_mean": 6.446424833939091e-06, "mean": 3.6914688479328106e-06, "normalized": [ 1.5294867291228127, 0.63348823892386, 1.1487447413186276, 0.5666523794690017, 0.30417708299540647, 0.347931809665085, 0.480405579791847, 0.5643067429040126, 0.6040226655402136, 0.5371983242313995, 0.7139156338995314, 0.9519107124097672, 0.7033642113963686, 0.7309137073086721, 0.64465215578137, 0.26016492268192315, 0.27317744038515385, 0.38409968136196165, 0.5564685521294294, 0.6459697208189646, 1.1764134223250242, 1.3443189191088418, 1.4240704391297414, 1.5037184806193353, 1.5204890244483313, 1.618553411169592, 1.865181309342729, 1.5801984775975157, 2.316915113115335, 1.8792787809683902, 1.727277195313884, 1.4625343947258718 ], "peak_layer": 29, "peak_normalized": 2.316915113115335, "population_cv": 0.5613822274727525, "rest_mean": 3.422263657908102e-06, "spike_contrast": 1.5034425714619797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.145176874066237e-06, "top_five_layers": [ 29, 30, 27, 31, 26 ], "top_five_spike_overlap": 0 }, "values": [ 5.646052613883512e-06, 2.3385020995192463e-06, 4.240555426804349e-06, 2.0917796064168215e-06, 1.122860226132616e-06, 1.2843794365835492e-06, 1.7734022321747034e-06, 2.0831207621085923e-06, 2.2297308532870375e-06, 1.9830508790619206e-06, 2.6353973225923255e-06, 3.5139487408741843e-06, 2.5964470751205226e-06, 2.6981451810570434e-06, 2.3797133508196566e-06, 9.603907074051676e-07, 1.008426011139818e-06, 1.4178920082486002e-06, 2.054186325040064e-06, 2.384577101111063e-06, 4.342693500802852e-06, 4.9625114115769975e-06, 5.256911663309438e-06, 5.550929927267134e-06, 5.612837867374765e-06, 5.9748394960479345e-06, 6.885258699185215e-06, 5.833253453602083e-06, 8.552819963369984e-06, 6.93729907652596e-06, 6.376189958245959e-06, 5.398900157160824e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.0313041968856851e-05, "first_to_last_ratio": 0.39957890355691367, "imbalance_abs_log_ratio": 0.9173440275030031, "last_quartile_mean": 2.5809775934249046e-05, "mean": 1.4741760296033135e-05, "normalized": [ 1.5558178418705222, 0.6371812985554408, 1.1430115844766933, 0.5651347220620626, 0.30571071286989043, 0.3485567104343656, 0.4774182631957655, 0.563809572533981, 0.6002213549646791, 0.5340330387924979, 0.7130055907666595, 0.9511132257714032, 0.6924732073436101, 0.7238195117723928, 0.6406610268334267, 0.24972491904320285, 0.2665104960045955, 0.3806326830724979, 0.5562541410983002, 0.6421888448376827, 1.1712220492382042, 1.3430540694729625, 1.426395171973413, 1.5057031409504333, 1.5356562461314327, 1.6199050072685577, 1.8779968028411653, 1.5872014101530674, 2.3447342214096167, 1.8683154808873674, 1.7248296546827906, 1.4477079986913193 ], "peak_layer": 29, "peak_normalized": 2.3447342214096167, "population_cv": 0.5659781309990658, "rest_mean": 1.3659589142291764e-05, "spike_contrast": 1.5070354102014207, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0585484526236543e-05, "top_five_layers": [ 29, 27, 30, 31, 26 ], "top_five_spike_overlap": 0 }, "values": [ 2.2935493689146824e-05, 9.393173968419433e-06, 1.685000279394444e-05, 8.331080607604235e-06, 4.5067140490573365e-06, 5.138339474797249e-06, 7.037985596980434e-06, 8.311545570904855e-06, 8.848319339449517e-06, 7.872587048041169e-06, 1.051095750881359e-05, 1.4021083188708872e-05, 1.0208274034084752e-05, 1.0670373740140349e-05, 9.44447128858883e-06, 3.681384896481177e-06, 3.928833848476643e-06, 5.611195774690714e-06, 8.200165211746935e-06, 9.466994015383534e-06, 1.7265874703298323e-05, 1.9798981156782247e-05, 2.1027575712651014e-05, 2.219671478087548e-05, 2.2638276277575642e-05, 2.3880251319496892e-05, 2.7684978704201058e-05, 2.339814273000229e-05, 3.456550984992646e-05, 2.7542258976609446e-05, 2.5427025320823304e-05, 2.134176429535728e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.009145708754658699, "first_to_last_ratio": 0.39957889743135716, "imbalance_abs_log_ratio": 0.9173440428330331, "last_quartile_mean": 0.02288836776278913, "mean": 0.013073140537017025, "normalized": [ 1.5558178691361477, 0.6371812873883458, 1.143011598762468, 0.5651346825162038, 0.3057107042206736, 0.3485567100234092, 0.47741826112733665, 0.5638095585980932, 0.6002213650547769, 0.5340330246694877, 0.7130055747038815, 0.9511132667062547, 0.692473171053213, 0.7238195005894487, 0.6406609754578104, 0.24972491326456617, 0.2665104811282862, 0.3806326734056363, 0.5562541580916099, 0.6421888466836433, 1.1712220415317927, 1.3430540180354118, 1.4263952722058089, 1.505703094517941, 1.5356562665173468, 1.6199051105332753, 1.877996734112475, 1.5872013856346572, 2.34473428701603, 1.8683154433096476, 1.7248296206158347, 1.4477081033884864 ], "peak_layer": 29, "peak_normalized": 2.34473428701603, "population_cv": 0.5659781409949523, "rest_mean": 0.012113460322359094, "spike_contrast": 1.507035414354221, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.018255413696169853, "top_five_layers": [ 29, 27, 30, 31, 26 ], "top_five_spike_overlap": 0 }, "values": [ 0.020339425653219223, 0.008329960517585278, 0.01494275126606226, 0.007388085126876831, 0.0039965989999473095, 0.004556730855256319, 0.006241356022655964, 0.0073707615956664085, 0.007846778258681297, 0.006981488782912493, 0.009321222081780434, 0.012434037402272224, 0.009052799083292484, 0.00946259405463934, 0.00837545096874237, 0.0032646888867020607, 0.003484128974378109, 0.004976064432412386, 0.007271988783031702, 0.008395425044000149, 0.015311550348997116, 0.017557933926582336, 0.018647465854883194, 0.019684268161654472, 0.020075850188732147, 0.021177247166633606, 0.02455131523311138, 0.020749706774950027, 0.03065304085612297, 0.024424750357866287, 0.022548940032720566, 0.018926091492176056 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.574768711686829e-07, "first_to_last_ratio": 0.3822604187682079, "imbalance_abs_log_ratio": 0.9616531781200647, "last_quartile_mean": 1.7199710953264002e-06, "mean": 9.503018119083606e-07, "normalized": [ 1.710462287653802, 0.6106596152780777, 1.1204318353188867, 0.578372698668433, 0.26243500094691574, 0.3526857345218465, 0.40843249489314853, 0.4914092526788544, 0.4957557052633209, 0.40719059148141235, 0.6062004989292337, 0.8649403856141179, 0.6653960206485303, 0.7108456658711597, 0.6350704165794606, 0.1774114383534533, 0.2713061284073076, 0.34448335189062385, 0.5523846644819398, 0.6344543697998888, 1.205861295517406, 1.3960194260372356, 1.458471938449579, 1.5599521119745998, 1.5844376236777387, 1.6451437430639262, 1.8959829898090674, 1.626547492622029, 2.4008070962757677, 1.9696132310887802, 1.813011017317309, 1.543823876886148 ], "peak_layer": 29, "peak_normalized": 2.4008070962757677, "population_cv": 0.6132533733917195, "rest_mean": 8.727028973385973e-07, "spike_contrast": 1.569074601174147, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3693359505850822e-06, "top_five_layers": [ 29, 30, 27, 31, 1 ], "top_five_spike_overlap": 0 }, "values": [ 1.6254554111583275e-06, 5.803109388580197e-07, 1.0647484032233479e-06, 5.496286235029402e-07, 2.4939245690802636e-07, 3.351578925503418e-07, 3.881341399392113e-07, 4.6698710320924874e-07, 4.7111754497564107e-07, 3.869539568768232e-07, 5.76073432512203e-07, 8.219544156418124e-07, 6.323270440589113e-07, 6.755179242645681e-07, 6.035085675648588e-07, 1.6859441132055508e-07, 2.578227054073068e-07, 3.273631534739252e-07, 5.249321475275792e-07, 6.029231371940114e-07, 1.1459321740403539e-06, 1.3266397900224547e-06, 1.385988525726134e-06, 1.482425318499736e-06, 1.5056939446367323e-06, 1.5633830798833515e-06, 1.8017560705629876e-06, 1.5457110293937149e-06, 2.281491333633312e-06, 1.8717270222623483e-06, 1.722907654766459e-06, 1.4670986274722964e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.837243347490585e-06, "first_to_last_ratio": 0.40636222975912695, "imbalance_abs_log_ratio": 0.9005103256217483, "last_quartile_mean": 1.6825489296934393e-05, "mean": 9.834350791493307e-06, "normalized": [ 1.5251245589223423, 0.6268534751803682, 1.1358978319631863, 0.5342766168184842, 0.32156135683620435, 0.34603480684096316, 0.5056955121723139, 0.5664834091985615, 0.6366995092986562, 0.5714340782015497, 0.7569982736970645, 0.9928803137356949, 0.7186055380199308, 0.7479928031827885, 0.6849140027219738, 0.21832989932248908, 0.2793170013221158, 0.4130909093851681, 0.5875122963256634, 0.6732858979284184, 1.1886843754660874, 1.3704808317703552, 1.427662092234709, 1.4830671545938088, 1.595028118071864, 1.627225338625226, 1.8396645878168607, 1.5251907756173002, 2.3393243002734927, 1.8118900136130478, 1.6024649187818438, 1.3463294020614671 ], "peak_layer": 29, "peak_normalized": 2.3393243002734927, "population_cv": 0.5475043977949907, "rest_mean": 9.082233275505672e-06, "spike_contrast": 1.5299965279798284, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3895785377826541e-05, "top_five_layers": [ 29, 27, 30, 26, 31 ], "top_five_spike_overlap": 0 }, "values": [ 1.4998609913163818e-05, 6.164696969790384e-06, 1.1170817742822692e-05, 5.254263669485226e-06, 3.162347184115788e-06, 3.4030276765406597e-06, 4.973187060386408e-06, 5.570996563619701e-06, 6.261526323214639e-06, 5.619683179247659e-06, 7.444586572091794e-06, 9.764333299244754e-06, 7.06701894159778e-06, 7.356023616011953e-06, 6.735684564773692e-06, 2.1471328182087746e-06, 2.7469013730296865e-06, 4.062480911670718e-06, 5.7778020163823385e-06, 6.6213297031936236e-06, 1.1689939128700644e-05, 1.3477789252647199e-05, 1.40401298267534e-05, 1.4585002645617351e-05, 1.5686066035414115e-05, 1.6002704796846956e-05, 1.8091906895278953e-05, 1.4999261111370288e-05, 2.300573578395415e-05, 1.7818761989474297e-05, 1.5759202142362483e-05, 1.32402756207739e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.331668606027961e-06, "first_to_last_ratio": 0.37652711875536754, "imbalance_abs_log_ratio": 0.9767652057694082, "last_quartile_mean": 1.1504267263262591e-05, "mean": 6.6139987815461154e-06, "normalized": [ 1.43041458747986, 0.5915824778695964, 1.0628524255756664, 0.4720208928952709, 0.3249571951503614, 0.32530495936634546, 0.5021667837664909, 0.5300951792850189, 0.6375122437915461, 0.5808919253486969, 0.7210798198798327, 0.97702603221025, 0.717856778528966, 0.720609258730328, 0.7009724751910035, 0.1867600754297654, 0.2849012445298506, 0.44885820098819357, 0.6203057532024762, 0.7121274696413161, 1.1940331031533082, 1.4444483655844333, 1.424043860567118, 1.4741267205385535, 1.7485546276719308, 1.7169348985773352, 1.9028313045331813, 1.5326281147396288, 2.428156586134552, 1.8387406559537405, 1.4841764050766164, 1.263029578608766 ], "peak_layer": 29, "peak_normalized": 2.428156586134552, "population_cv": 0.5648545726504486, "rest_mean": 6.054207885965836e-06, "spike_contrast": 1.5917639101919008, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.636869617679621e-06, "top_five_layers": [ 29, 27, 30, 25, 26 ], "top_five_spike_overlap": 1 }, "values": [ 9.460760338697582e-06, 3.912725787813542e-06, 7.029704647720791e-06, 3.121945610473631e-06, 2.149266492779134e-06, 2.1515666048799176e-06, 3.321330495964503e-06, 3.5060488698945846e-06, 4.216505203658016e-06, 3.842018486466259e-06, 4.769221050082706e-06, 6.462048986577429e-06, 4.747903858515201e-06, 4.766108759213239e-06, 4.636231096810661e-06, 1.2352309113339288e-06, 1.8843364841814036e-06, 2.9687475944228936e-06, 4.1027014958672225e-06, 4.710010216513183e-06, 7.897333489381708e-06, 9.55357972998172e-06, 9.418624358659144e-06, 9.749872333486564e-06, 1.1564938176888973e-05, 1.1355805327184498e-05, 1.2585323929670267e-05, 1.0136800483451225e-05, 1.6059824702097103e-05, 1.2161428458057344e-05, 9.816340934776235e-06, 8.353676093975082e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.1292764813551912e-05, "first_to_last_ratio": 0.37783048366494065, "imbalance_abs_log_ratio": 0.9733096398618312, "last_quartile_mean": 5.635533852910157e-05, "mean": 3.159250317708029e-05, "normalized": [ 1.482066067786081, 0.6128089902703769, 1.1079572882309123, 0.5339964494498499, 0.29044186792909177, 0.3352417717243252, 0.469628384430032, 0.5597117204362547, 0.5936182437594482, 0.5198266141043868, 0.7058453157249553, 0.9273089878310177, 0.6699840087338256, 0.7439837778533973, 0.6502527329579387, 0.22627068614548218, 0.262454163087834, 0.36907877128447675, 0.5613139624883691, 0.6571387234246169, 1.1835844028068934, 1.3363166236750326, 1.4226757890468276, 1.5079347804524084, 1.5296526797645231, 1.6193459917783275, 1.941711012573152, 1.6212855176431946, 2.3766952020840555, 1.9054387812782299, 1.7892038011178224, 1.4872268901268606 ], "peak_layer": 29, "peak_normalized": 2.3766952020840555, "population_cv": 0.5799347827395998, "rest_mean": 2.9275527392908567e-05, "spike_contrast": 1.5065201668165664, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.410417241160758e-05, "top_five_layers": [ 29, 27, 30, 31, 28 ], "top_five_spike_overlap": 0 }, "values": [ 4.6822176955174655e-05, 1.9360169972060248e-05, 3.500314414850436e-05, 1.6870284525793977e-05, 9.175785635306966e-06, 1.0591126738290768e-05, 1.4836736227152869e-05, 1.7682694306131452e-05, 1.8753886251943186e-05, 1.6422623957623728e-05, 2.229942037956789e-05, 2.9296012144186534e-05, 2.1166471924516372e-05, 2.350430986552965e-05, 2.054311153187882e-05, 7.148457370931283e-06, 8.291583981190342e-06, 1.166012225439772e-05, 1.7733313143253326e-05, 2.0760657207574695e-05, 3.739239400601946e-05, 4.2217587179038674e-05, 4.4945889385417104e-05, 4.763943434227258e-05, 4.832555714529008e-05, 5.115919339004904e-05, 6.13435113336891e-05, 5.1220467867096886e-05, 7.5085750722792e-05, 6.019758075126447e-05, 5.6525426771258935e-05, 4.6985220251372084e-05 ] } } } } } }, "step": 8000 } ], "environment": { "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", "compile": false, "compute_capability": [ 12, 0 ], "cublas_workspace_config": ":4096:8", "cuda": "12.8", "deterministic_algorithms": true, "gpu": "NVIDIA GeForce RTX 5090", "python": "3.10.14", "torch": "2.11.0+cu128" }, "evaluations": [ { "bits_per_byte": 8.034013546791773, "cross_entropy_nats": 5.5687538385391235, "step": 0 }, { "bits_per_byte": 5.132431212443787, "cross_entropy_nats": 3.5575302243232727, "step": 100 }, { "bits_per_byte": 3.0911456537212687, "cross_entropy_nats": 2.1426188945770264, "step": 500 }, { "bits_per_byte": 1.947918907793682, "cross_entropy_nats": 1.3501944988965988, "step": 2000 }, { "bits_per_byte": 1.7911587423283797, "cross_entropy_nats": 1.241536632180214, "step": 4000 }, { "bits_per_byte": 1.7093240655856785, "cross_entropy_nats": 1.1848131567239761, "step": 8000 } ], "hashes": { "final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d", "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", "final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9", "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", "initial_public_parameter_elements": 18985152, "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", "initial_public_parameter_tensors": 227, "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" }, "manifest": { "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", "input_gate_tensor_hashes": { "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" }, "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", "parent_path": "experiments/k3/attnres_gradient/manifest.json", "path": "experiments/k3/attnres_spike/manifest.json", "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" }, "model": { "aggregation_groups": 8, "blocks_per_group": 4, "d_ff": 768, "d_model": 192, "heads": 6, "layers": 32, "parameters": { "core": 18985152, "embedding": 98304, "mixer": 24960, "total": 19010112 } }, "optimizer": { "betas": [ 0.9, 0.95 ], "epsilon": 1e-08, "grad_clip": 1.0, "min_lr": 3e-05, "name": "AdamW", "peak_lr": 0.0003, "warmup_steps": 400, "weight_decay_ndim_ge_2": 0.1 }, "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", "round05_equivalence": { "checks": { "evaluations": true, "final_model_state": true, "final_optimizer_state": true, "post_mlp_element_rms_all_steps": true, "training_history": true }, "passed": true }, "run_kind": "formal", "schema_version": 1, "seed": 2026073002, "steps": 8000, "target_bytes_seen": 65536000, "timing": { "mean_ms": 223.82424076982844, "measured_steps": 7980, "median_ms": 227.36876149429008, "p95_ms": 229.48265075683594, "peak_allocated_bytes": 12841009152, "peak_reserved_bytes": 13922992128, "warmup_steps_excluded": 20 }, "training_history": [ { "bits_per_byte": 8.018870646365954, "learning_rate": 7.499999999999999e-07, "loss_nats": 5.558257579803467, "step": 1, "unclipped_grad_norm": 26.834381103515625 }, { "bits_per_byte": 7.214299345307752, "learning_rate": 7.499999999999999e-06, "loss_nats": 5.000571250915527, "step": 10, "unclipped_grad_norm": 12.740906715393066 }, { "bits_per_byte": 6.684460697974439, "learning_rate": 1.4999999999999999e-05, "loss_nats": 4.633315086364746, "step": 20, "unclipped_grad_norm": 3.454355239868164 }, { "bits_per_byte": 6.541223915307444, "learning_rate": 2.2499999999999998e-05, "loss_nats": 4.534030914306641, "step": 30, "unclipped_grad_norm": 3.060823440551758 }, { "bits_per_byte": 6.411011034824987, "learning_rate": 2.9999999999999997e-05, "loss_nats": 4.443774223327637, "step": 40, "unclipped_grad_norm": 3.0002496242523193 }, { "bits_per_byte": 6.085588902515397, "learning_rate": 3.75e-05, "loss_nats": 4.2182087898254395, "step": 50, "unclipped_grad_norm": 2.976402997970581 }, { "bits_per_byte": 5.879418164452873, "learning_rate": 4.4999999999999996e-05, "loss_nats": 4.0753021240234375, "step": 60, "unclipped_grad_norm": 2.708517551422119 }, { "bits_per_byte": 5.710178980836628, "learning_rate": 5.2499999999999995e-05, "loss_nats": 3.9579944610595703, "step": 70, "unclipped_grad_norm": 2.559852361679077 }, { "bits_per_byte": 5.512089019607543, "learning_rate": 5.9999999999999995e-05, "loss_nats": 3.8206889629364014, "step": 80, "unclipped_grad_norm": 2.5563650131225586 }, { "bits_per_byte": 5.341444737733749, "learning_rate": 6.749999999999999e-05, "loss_nats": 3.7024073600769043, "step": 90, "unclipped_grad_norm": 2.3861565589904785 }, { "bits_per_byte": 5.169528816139475, "learning_rate": 7.5e-05, "loss_nats": 3.5832443237304688, "step": 100, "unclipped_grad_norm": 2.241894006729126 }, { "bits_per_byte": 5.000991665704052, "learning_rate": 8.249999999999999e-05, "loss_nats": 3.466423273086548, "step": 110, "unclipped_grad_norm": 2.1930899620056152 }, { "bits_per_byte": 4.733823840860015, "learning_rate": 8.999999999999999e-05, "loss_nats": 3.2812366485595703, "step": 120, "unclipped_grad_norm": 1.8853988647460938 }, { "bits_per_byte": 4.585053344418857, "learning_rate": 9.75e-05, "loss_nats": 3.178116798400879, "step": 130, "unclipped_grad_norm": 1.6627076864242554 }, { "bits_per_byte": 4.391709416422983, "learning_rate": 0.00010499999999999999, "loss_nats": 3.0441009998321533, "step": 140, "unclipped_grad_norm": 1.4586857557296753 }, { "bits_per_byte": 4.260228335873064, "learning_rate": 0.0001125, "loss_nats": 2.952965259552002, "step": 150, "unclipped_grad_norm": 1.235063076019287 }, { "bits_per_byte": 4.153197684831009, "learning_rate": 0.00011999999999999999, "loss_nats": 2.878777265548706, "step": 160, "unclipped_grad_norm": 1.0407519340515137 }, { "bits_per_byte": 3.997309514401244, "learning_rate": 0.00012749999999999998, "loss_nats": 2.770723819732666, "step": 170, "unclipped_grad_norm": 0.9049285054206848 }, { "bits_per_byte": 3.860879805055398, "learning_rate": 0.00013499999999999997, "loss_nats": 2.6761579513549805, "step": 180, "unclipped_grad_norm": 1.0858614444732666 }, { "bits_per_byte": 3.870312021559318, "learning_rate": 0.0001425, "loss_nats": 2.6826958656311035, "step": 190, "unclipped_grad_norm": 0.6896197199821472 }, { "bits_per_byte": 3.747811875037455, "learning_rate": 0.00015, "loss_nats": 2.597785234451294, "step": 200, "unclipped_grad_norm": 0.712465226650238 }, { "bits_per_byte": 3.69472358243852, "learning_rate": 0.0001575, "loss_nats": 2.5609872341156006, "step": 210, "unclipped_grad_norm": 0.8807040452957153 }, { "bits_per_byte": 3.662194783854328, "learning_rate": 0.00016499999999999997, "loss_nats": 2.538439989089966, "step": 220, "unclipped_grad_norm": 0.7631296515464783 }, { "bits_per_byte": 3.6752317567203328, "learning_rate": 0.0001725, "loss_nats": 2.5474765300750732, "step": 230, "unclipped_grad_norm": 0.6869579553604126 }, { "bits_per_byte": 3.6565805821995627, "learning_rate": 0.00017999999999999998, "loss_nats": 2.53454852104187, "step": 240, "unclipped_grad_norm": 0.781292736530304 }, { "bits_per_byte": 3.5245367744160205, "learning_rate": 0.0001875, "loss_nats": 2.4430227279663086, "step": 250, "unclipped_grad_norm": 1.0127525329589844 }, { "bits_per_byte": 3.524651658826797, "learning_rate": 0.000195, "loss_nats": 2.4431023597717285, "step": 260, "unclipped_grad_norm": 0.8615073561668396 }, { "bits_per_byte": 3.485441678221887, "learning_rate": 0.00020249999999999996, "loss_nats": 2.415924072265625, "step": 270, "unclipped_grad_norm": 0.7731815576553345 }, { "bits_per_byte": 3.4956767097400343, "learning_rate": 0.00020999999999999998, "loss_nats": 2.423018455505371, "step": 280, "unclipped_grad_norm": 0.9061474800109863 }, { "bits_per_byte": 3.3938416545805445, "learning_rate": 0.00021749999999999997, "loss_nats": 2.3524317741394043, "step": 290, "unclipped_grad_norm": 1.0149996280670166 }, { "bits_per_byte": 3.483576010425327, "learning_rate": 0.000225, "loss_nats": 2.414630889892578, "step": 300, "unclipped_grad_norm": 1.1414116621017456 }, { "bits_per_byte": 3.381058528107214, "learning_rate": 0.00023249999999999996, "loss_nats": 2.343571186065674, "step": 310, "unclipped_grad_norm": 1.1472890377044678 }, { "bits_per_byte": 3.3707543595634197, "learning_rate": 0.00023999999999999998, "loss_nats": 2.3364288806915283, "step": 320, "unclipped_grad_norm": 1.2828246355056763 }, { "bits_per_byte": 3.3462052120139605, "learning_rate": 0.0002475, "loss_nats": 2.3194127082824707, "step": 330, "unclipped_grad_norm": 0.9300209879875183 }, { "bits_per_byte": 3.3510090314178607, "learning_rate": 0.00025499999999999996, "loss_nats": 2.322742462158203, "step": 340, "unclipped_grad_norm": 0.7300398945808411 }, { "bits_per_byte": 3.324897249502662, "learning_rate": 0.0002625, "loss_nats": 2.304643154144287, "step": 350, "unclipped_grad_norm": 0.96440589427948 }, { "bits_per_byte": 3.301325651340707, "learning_rate": 0.00026999999999999995, "loss_nats": 2.288304567337036, "step": 360, "unclipped_grad_norm": 1.186806321144104 }, { "bits_per_byte": 3.330423052074884, "learning_rate": 0.00027749999999999997, "loss_nats": 2.3084733486175537, "step": 370, "unclipped_grad_norm": 1.40411376953125 }, { "bits_per_byte": 3.351273196769586, "learning_rate": 0.000285, "loss_nats": 2.322925567626953, "step": 380, "unclipped_grad_norm": 0.8571750521659851 }, { "bits_per_byte": 3.2969108566930587, "learning_rate": 0.0002925, "loss_nats": 2.2852444648742676, "step": 390, "unclipped_grad_norm": 1.172652006149292 }, { "bits_per_byte": 3.2316042286462263, "learning_rate": 0.0003, "loss_nats": 2.2399773597717285, "step": 400, "unclipped_grad_norm": 1.0324530601501465 }, { "bits_per_byte": 3.191360632309631, "learning_rate": 0.0002999988466112388, "loss_nats": 2.212082624435425, "step": 410, "unclipped_grad_norm": 0.9553669691085815 }, { "bits_per_byte": 3.2040835648551598, "learning_rate": 0.00029999538646466355, "loss_nats": 2.2209014892578125, "step": 420, "unclipped_grad_norm": 1.0212959051132202 }, { "bits_per_byte": 3.1886670400318176, "learning_rate": 0.0002999896196193985, "loss_nats": 2.2102155685424805, "step": 430, "unclipped_grad_norm": 0.8366522789001465 }, { "bits_per_byte": 3.2100664973133752, "learning_rate": 0.00029998154617398326, "loss_nats": 2.225048542022705, "step": 440, "unclipped_grad_norm": 1.1677532196044922 }, { "bits_per_byte": 3.1886219805772913, "learning_rate": 0.0002999711662663707, "loss_nats": 2.210184335708618, "step": 450, "unclipped_grad_norm": 1.3463479280471802 }, { "bits_per_byte": 3.136037597145384, "learning_rate": 0.0002999584800739247, "loss_nats": 2.1737356185913086, "step": 460, "unclipped_grad_norm": 1.704139232635498 }, { "bits_per_byte": 3.1589835224235, "learning_rate": 0.0002999434878134173, "loss_nats": 2.189640522003174, "step": 470, "unclipped_grad_norm": 1.1849488019943237 }, { "bits_per_byte": 3.1210654754918714, "learning_rate": 0.000299926189741025, "loss_nats": 2.163357734680176, "step": 480, "unclipped_grad_norm": 0.8924318552017212 }, { "bits_per_byte": 3.0823876092087312, "learning_rate": 0.000299906586152324, "loss_nats": 2.1365482807159424, "step": 490, "unclipped_grad_norm": 1.3279240131378174 }, { "bits_per_byte": 3.12389562199447, "learning_rate": 0.0002998846773822856, "loss_nats": 2.1653194427490234, "step": 500, "unclipped_grad_norm": 1.3776742219924927 }, { "bits_per_byte": 3.102860080002138, "learning_rate": 0.0002998604638052702, "loss_nats": 2.1507387161254883, "step": 510, "unclipped_grad_norm": 1.061196208000183 }, { "bits_per_byte": 2.961672642602824, "learning_rate": 0.0002998339458350211, "loss_nats": 2.05287504196167, "step": 520, "unclipped_grad_norm": 1.0864008665084839 }, { "bits_per_byte": 3.0521231342057993, "learning_rate": 0.0002998051239246572, "loss_nats": 2.115570545196533, "step": 530, "unclipped_grad_norm": 0.9000556468963623 }, { "bits_per_byte": 2.992112883874938, "learning_rate": 0.00029977399856666537, "loss_nats": 2.073974609375, "step": 540, "unclipped_grad_norm": 0.8655318021774292 }, { "bits_per_byte": 2.9456885750316344, "learning_rate": 0.0002997405702928922, "loss_nats": 2.0417957305908203, "step": 550, "unclipped_grad_norm": 0.8011268973350525 }, { "bits_per_byte": 2.9526635034200264, "learning_rate": 0.0002997048396745345, "loss_nats": 2.046630382537842, "step": 560, "unclipped_grad_norm": 1.2049078941345215 }, { "bits_per_byte": 2.943776471919342, "learning_rate": 0.0002996668073221302, "loss_nats": 2.0404703617095947, "step": 570, "unclipped_grad_norm": 0.9911091923713684 }, { "bits_per_byte": 2.836173462615215, "learning_rate": 0.000299626473885547, "loss_nats": 1.9658856391906738, "step": 580, "unclipped_grad_norm": 0.9621384143829346 }, { "bits_per_byte": 2.898327304706431, "learning_rate": 0.00029958384005397226, "loss_nats": 2.008967399597168, "step": 590, "unclipped_grad_norm": 1.0092018842697144 }, { "bits_per_byte": 2.807946982025008, "learning_rate": 0.0002995389065559004, "loss_nats": 1.9463205337524414, "step": 600, "unclipped_grad_norm": 1.041255235671997 }, { "bits_per_byte": 2.8662821213364653, "learning_rate": 0.000299491674159121, "loss_nats": 1.98675537109375, "step": 610, "unclipped_grad_norm": 1.0585267543792725 }, { "bits_per_byte": 2.7525066546928727, "learning_rate": 0.00029944214367070535, "loss_nats": 1.9078922271728516, "step": 620, "unclipped_grad_norm": 0.9587937593460083 }, { "bits_per_byte": 2.7676626258001575, "learning_rate": 0.00029939031593699284, "loss_nats": 1.9183975458145142, "step": 630, "unclipped_grad_norm": 0.9836012721061707 }, { "bits_per_byte": 2.7750938241612886, "learning_rate": 0.00029933619184357654, "loss_nats": 1.9235484600067139, "step": 640, "unclipped_grad_norm": 0.9689151644706726 }, { "bits_per_byte": 2.6921524390602856, "learning_rate": 0.00029927977231528774, "loss_nats": 1.8660578727722168, "step": 650, "unclipped_grad_norm": 1.0390667915344238 }, { "bits_per_byte": 2.6851900654054064, "learning_rate": 0.00029922105831618057, "loss_nats": 1.8612319231033325, "step": 660, "unclipped_grad_norm": 1.6513670682907104 }, { "bits_per_byte": 2.6846809967588525, "learning_rate": 0.0002991600508495153, "loss_nats": 1.8608790636062622, "step": 670, "unclipped_grad_norm": 0.9175130724906921 }, { "bits_per_byte": 2.7078205745018398, "learning_rate": 0.00029909675095774114, "loss_nats": 1.8769181966781616, "step": 680, "unclipped_grad_norm": 1.0189234018325806 }, { "bits_per_byte": 2.5875252455640383, "learning_rate": 0.0002990311597224787, "loss_nats": 1.793535828590393, "step": 690, "unclipped_grad_norm": 0.9444974064826965 }, { "bits_per_byte": 2.5413828162854992, "learning_rate": 0.00029896327826450103, "loss_nats": 1.761552333831787, "step": 700, "unclipped_grad_norm": 1.1904406547546387 }, { "bits_per_byte": 2.623998466245768, "learning_rate": 0.0002988931077437152, "loss_nats": 1.818817138671875, "step": 710, "unclipped_grad_norm": 1.1858620643615723 }, { "bits_per_byte": 2.683836561943116, "learning_rate": 0.00029882064935914164, "loss_nats": 1.8602937459945679, "step": 720, "unclipped_grad_norm": 0.9852609634399414 }, { "bits_per_byte": 2.4849188482154285, "learning_rate": 0.0002987459043488943, "loss_nats": 1.722414493560791, "step": 730, "unclipped_grad_norm": 1.1975579261779785 }, { "bits_per_byte": 2.5244201674309004, "learning_rate": 0.0002986688739901591, "loss_nats": 1.7497947216033936, "step": 740, "unclipped_grad_norm": 0.9646193385124207 }, { "bits_per_byte": 2.5296962511940695, "learning_rate": 0.0002985895595991723, "loss_nats": 1.7534518241882324, "step": 750, "unclipped_grad_norm": 1.191988468170166 }, { "bits_per_byte": 2.4988650933565446, "learning_rate": 0.00029850796253119824, "loss_nats": 1.7320812940597534, "step": 760, "unclipped_grad_norm": 1.092863917350769 }, { "bits_per_byte": 2.4838112799438723, "learning_rate": 0.00029842408418050544, "loss_nats": 1.721646785736084, "step": 770, "unclipped_grad_norm": 1.0348618030548096 }, { "bits_per_byte": 2.4223032327066463, "learning_rate": 0.0002983379259803436, "loss_nats": 1.679012656211853, "step": 780, "unclipped_grad_norm": 1.034103274345398 }, { "bits_per_byte": 2.485370818621896, "learning_rate": 0.0002982494894029186, "loss_nats": 1.7227277755737305, "step": 790, "unclipped_grad_norm": 1.060265302658081 }, { "bits_per_byte": 2.4233009040643014, "learning_rate": 0.0002981587759593675, "loss_nats": 1.679704189300537, "step": 800, "unclipped_grad_norm": 1.3169445991516113 }, { "bits_per_byte": 2.4599308009409677, "learning_rate": 0.00029806578719973283, "loss_nats": 1.7050940990447998, "step": 810, "unclipped_grad_norm": 0.9821087121963501 }, { "bits_per_byte": 2.425854846429614, "learning_rate": 0.00029797052471293596, "loss_nats": 1.6814744472503662, "step": 820, "unclipped_grad_norm": 1.0924170017242432 }, { "bits_per_byte": 2.4277851797028984, "learning_rate": 0.0002978729901267499, "loss_nats": 1.6828124523162842, "step": 830, "unclipped_grad_norm": 1.055789828300476 }, { "bits_per_byte": 2.3982261775338056, "learning_rate": 0.0002977731851077716, "loss_nats": 1.6623237133026123, "step": 840, "unclipped_grad_norm": 1.230089545249939 }, { "bits_per_byte": 2.339956047664374, "learning_rate": 0.0002976711113613935, "loss_nats": 1.621933937072754, "step": 850, "unclipped_grad_norm": 0.94926518201828 }, { "bits_per_byte": 2.3803994878412724, "learning_rate": 0.0002975667706317743, "loss_nats": 1.6499671936035156, "step": 860, "unclipped_grad_norm": 1.1705771684646606 }, { "bits_per_byte": 2.3436123988217177, "learning_rate": 0.00029746016470180915, "loss_nats": 1.6244683265686035, "step": 870, "unclipped_grad_norm": 1.1496235132217407 }, { "bits_per_byte": 2.3796778486382517, "learning_rate": 0.00029735129539309926, "loss_nats": 1.6494669914245605, "step": 880, "unclipped_grad_norm": 1.1764546632766724 }, { "bits_per_byte": 2.357840007579645, "learning_rate": 0.0002972401645659207, "loss_nats": 1.634330153465271, "step": 890, "unclipped_grad_norm": 0.9926145672798157 }, { "bits_per_byte": 2.2929997964819524, "learning_rate": 0.00029712677411919283, "loss_nats": 1.5893863439559937, "step": 900, "unclipped_grad_norm": 1.009945034980774 }, { "bits_per_byte": 2.3411444477818364, "learning_rate": 0.0002970111259904454, "loss_nats": 1.6227576732635498, "step": 910, "unclipped_grad_norm": 1.2830723524093628 }, { "bits_per_byte": 2.2736926801307917, "learning_rate": 0.0002968932221557859, "loss_nats": 1.5760036706924438, "step": 920, "unclipped_grad_norm": 1.0900828838348389 }, { "bits_per_byte": 2.3038844064724087, "learning_rate": 0.0002967730646298655, "loss_nats": 1.596930980682373, "step": 930, "unclipped_grad_norm": 1.1090145111083984 }, { "bits_per_byte": 2.2389674911124313, "learning_rate": 0.0002966506554658449, "loss_nats": 1.551934003829956, "step": 940, "unclipped_grad_norm": 0.9897664189338684 }, { "bits_per_byte": 2.2434025797128814, "learning_rate": 0.00029652599675535876, "loss_nats": 1.5550081729888916, "step": 950, "unclipped_grad_norm": 1.0138952732086182 }, { "bits_per_byte": 2.2236732459538526, "learning_rate": 0.0002963990906284808, "loss_nats": 1.5413328409194946, "step": 960, "unclipped_grad_norm": 1.0875126123428345 }, { "bits_per_byte": 2.2667958318658914, "learning_rate": 0.0002962699392536863, "loss_nats": 1.5712231397628784, "step": 970, "unclipped_grad_norm": 1.245374321937561 }, { "bits_per_byte": 2.2344890628839638, "learning_rate": 0.0002961385448378161, "loss_nats": 1.5488297939300537, "step": 980, "unclipped_grad_norm": 1.1757959127426147 }, { "bits_per_byte": 2.1613678871472266, "learning_rate": 0.00029600490962603817, "loss_nats": 1.4981460571289062, "step": 990, "unclipped_grad_norm": 1.066755771636963 }, { "bits_per_byte": 2.207448574654106, "learning_rate": 0.00029586903590180956, "loss_nats": 1.5300867557525635, "step": 1000, "unclipped_grad_norm": 0.9658553004264832 }, { "bits_per_byte": 2.213320750285166, "learning_rate": 0.00029573092598683735, "loss_nats": 1.5341570377349854, "step": 1010, "unclipped_grad_norm": 1.1267611980438232 }, { "bits_per_byte": 2.2186436133293697, "learning_rate": 0.00029559058224103895, "loss_nats": 1.537846565246582, "step": 1020, "unclipped_grad_norm": 0.9853240251541138 }, { "bits_per_byte": 2.1280228589020402, "learning_rate": 0.0002954480070625017, "loss_nats": 1.4750330448150635, "step": 1030, "unclipped_grad_norm": 0.972481369972229 }, { "bits_per_byte": 2.2322469250646653, "learning_rate": 0.00029530320288744205, "loss_nats": 1.5472756624221802, "step": 1040, "unclipped_grad_norm": 1.0531573295593262 }, { "bits_per_byte": 2.179971250491057, "learning_rate": 0.0002951561721901638, "loss_nats": 1.5110409259796143, "step": 1050, "unclipped_grad_norm": 0.9985654950141907 }, { "bits_per_byte": 2.1476089311129005, "learning_rate": 0.00029500691748301594, "loss_nats": 1.4886090755462646, "step": 1060, "unclipped_grad_norm": 1.100019931793213 }, { "bits_per_byte": 2.2640807417567204, "learning_rate": 0.00029485544131634963, "loss_nats": 1.5693411827087402, "step": 1070, "unclipped_grad_norm": 1.0959177017211914 }, { "bits_per_byte": 2.1797610876917024, "learning_rate": 0.0002947017462784745, "loss_nats": 1.5108952522277832, "step": 1080, "unclipped_grad_norm": 1.088409423828125 }, { "bits_per_byte": 2.1080023585151006, "learning_rate": 0.00029454583499561476, "loss_nats": 1.461155891418457, "step": 1090, "unclipped_grad_norm": 1.0188393592834473 }, { "bits_per_byte": 2.1918825969071642, "learning_rate": 0.00029438771013186413, "loss_nats": 1.5192972421646118, "step": 1100, "unclipped_grad_norm": 0.9501848816871643 }, { "bits_per_byte": 2.205643788715952, "learning_rate": 0.0002942273743891401, "loss_nats": 1.5288357734680176, "step": 1110, "unclipped_grad_norm": 0.9797388911247253 }, { "bits_per_byte": 2.0930500148664373, "learning_rate": 0.00029406483050713824, "loss_nats": 1.4507917165756226, "step": 1120, "unclipped_grad_norm": 0.8856902122497559 }, { "bits_per_byte": 2.095190166973774, "learning_rate": 0.00029390008126328504, "loss_nats": 1.4522751569747925, "step": 1130, "unclipped_grad_norm": 0.9447507262229919 }, { "bits_per_byte": 2.116739593126884, "learning_rate": 0.00029373312947269045, "loss_nats": 1.4672120809555054, "step": 1140, "unclipped_grad_norm": 1.0488460063934326 }, { "bits_per_byte": 2.1023439573190634, "learning_rate": 0.00029356397798809995, "loss_nats": 1.4572337865829468, "step": 1150, "unclipped_grad_norm": 0.9838997721672058 }, { "bits_per_byte": 2.0562952585852448, "learning_rate": 0.0002933926296998457, "loss_nats": 1.425315260887146, "step": 1160, "unclipped_grad_norm": 0.9282211065292358 }, { "bits_per_byte": 2.160992449020393, "learning_rate": 0.0002932190875357972, "loss_nats": 1.497885823249817, "step": 1170, "unclipped_grad_norm": 1.0253955125808716 }, { "bits_per_byte": 2.0716469459492117, "learning_rate": 0.00029304335446131126, "loss_nats": 1.4359562397003174, "step": 1180, "unclipped_grad_norm": 0.9352066516876221 }, { "bits_per_byte": 2.0893013090255805, "learning_rate": 0.00029286543347918116, "loss_nats": 1.4481933116912842, "step": 1190, "unclipped_grad_norm": 1.0826658010482788 }, { "bits_per_byte": 2.0752424152481495, "learning_rate": 0.0002926853276295856, "loss_nats": 1.438448429107666, "step": 1200, "unclipped_grad_norm": 0.9942638874053955 }, { "bits_per_byte": 2.2049758081000004, "learning_rate": 0.0002925030399900368, "loss_nats": 1.5283727645874023, "step": 1210, "unclipped_grad_norm": 1.1482313871383667 }, { "bits_per_byte": 2.0506430487646385, "learning_rate": 0.00029231857367532726, "loss_nats": 1.4213974475860596, "step": 1220, "unclipped_grad_norm": 0.9142818450927734 }, { "bits_per_byte": 2.00269480165204, "learning_rate": 0.0002921319318374776, "loss_nats": 1.3881622552871704, "step": 1230, "unclipped_grad_norm": 0.9387761354446411 }, { "bits_per_byte": 2.0166881700394925, "learning_rate": 0.00029194311766568174, "loss_nats": 1.3978617191314697, "step": 1240, "unclipped_grad_norm": 1.0545530319213867 }, { "bits_per_byte": 2.1787518934964396, "learning_rate": 0.000291752134386253, "loss_nats": 1.5101957321166992, "step": 1250, "unclipped_grad_norm": 1.1504697799682617 }, { "bits_per_byte": 2.1047604855463367, "learning_rate": 0.00029155898526256856, "loss_nats": 1.4589087963104248, "step": 1260, "unclipped_grad_norm": 0.9371413588523865 }, { "bits_per_byte": 1.9885089846782722, "learning_rate": 0.0002913636735950141, "loss_nats": 1.3783293962478638, "step": 1270, "unclipped_grad_norm": 0.9821643233299255 }, { "bits_per_byte": 2.1346452228561636, "learning_rate": 0.0002911662027209272, "loss_nats": 1.4796233177185059, "step": 1280, "unclipped_grad_norm": 0.9382128715515137 }, { "bits_per_byte": 2.022505311222276, "learning_rate": 0.00029096657601454013, "loss_nats": 1.4018938541412354, "step": 1290, "unclipped_grad_norm": 0.9164487719535828 }, { "bits_per_byte": 2.050535903573151, "learning_rate": 0.00029076479688692266, "loss_nats": 1.4213231801986694, "step": 1300, "unclipped_grad_norm": 0.9040083885192871 }, { "bits_per_byte": 2.128253659619498, "learning_rate": 0.00029056086878592323, "loss_nats": 1.4751930236816406, "step": 1310, "unclipped_grad_norm": 1.045804738998413 }, { "bits_per_byte": 2.0627867437420613, "learning_rate": 0.00029035479519611045, "loss_nats": 1.4298148155212402, "step": 1320, "unclipped_grad_norm": 1.0330690145492554 }, { "bits_per_byte": 1.954123977343318, "learning_rate": 0.0002901465796387134, "loss_nats": 1.3544955253601074, "step": 1330, "unclipped_grad_norm": 0.9727330803871155 }, { "bits_per_byte": 1.9791288789175636, "learning_rate": 0.00028993622567156154, "loss_nats": 1.3718276023864746, "step": 1340, "unclipped_grad_norm": 0.966886043548584 }, { "bits_per_byte": 2.1481475807754027, "learning_rate": 0.0002897237368890237, "loss_nats": 1.4889824390411377, "step": 1350, "unclipped_grad_norm": 0.9399362802505493 }, { "bits_per_byte": 1.9898208683390537, "learning_rate": 0.0002895091169219469, "loss_nats": 1.3792387247085571, "step": 1360, "unclipped_grad_norm": 0.9223746061325073 }, { "bits_per_byte": 2.0666878262115826, "learning_rate": 0.00028929236943759425, "loss_nats": 1.4325188398361206, "step": 1370, "unclipped_grad_norm": 0.9086402654647827 }, { "bits_per_byte": 2.05495087020345, "learning_rate": 0.00028907349813958225, "loss_nats": 1.4243834018707275, "step": 1380, "unclipped_grad_norm": 0.954355776309967 }, { "bits_per_byte": 2.067550663170961, "learning_rate": 0.0002888525067678175, "loss_nats": 1.4331169128417969, "step": 1390, "unclipped_grad_norm": 0.8977138996124268 }, { "bits_per_byte": 2.0721085473841274, "learning_rate": 0.00028862939909843273, "loss_nats": 1.4362761974334717, "step": 1400, "unclipped_grad_norm": 0.8833977580070496 }, { "bits_per_byte": 1.9799110560136877, "learning_rate": 0.0002884041789437225, "loss_nats": 1.3723697662353516, "step": 1410, "unclipped_grad_norm": 0.9529625177383423 }, { "bits_per_byte": 2.038817521691312, "learning_rate": 0.0002881768501520778, "loss_nats": 1.4132006168365479, "step": 1420, "unclipped_grad_norm": 1.083083152770996 }, { "bits_per_byte": 2.0292872750764, "learning_rate": 0.0002879474166079205, "loss_nats": 1.4065947532653809, "step": 1430, "unclipped_grad_norm": 0.9355389475822449 }, { "bits_per_byte": 1.987968787171912, "learning_rate": 0.00028771588223163694, "loss_nats": 1.3779549598693848, "step": 1440, "unclipped_grad_norm": 0.8836045265197754 }, { "bits_per_byte": 2.0423230440638482, "learning_rate": 0.00028748225097951054, "loss_nats": 1.4156304597854614, "step": 1450, "unclipped_grad_norm": 0.924534022808075 }, { "bits_per_byte": 1.9816687187055055, "learning_rate": 0.000287246526843655, "loss_nats": 1.3735880851745605, "step": 1460, "unclipped_grad_norm": 0.8437807559967041 }, { "bits_per_byte": 1.9797782854072217, "learning_rate": 0.00028700871385194533, "loss_nats": 1.3722777366638184, "step": 1470, "unclipped_grad_norm": 0.9143778085708618 }, { "bits_per_byte": 1.8767507025466492, "learning_rate": 0.00028676881606794945, "loss_nats": 1.3008644580841064, "step": 1480, "unclipped_grad_norm": 0.8930306434631348 }, { "bits_per_byte": 2.001770738868954, "learning_rate": 0.00028652683759085854, "loss_nats": 1.387521743774414, "step": 1490, "unclipped_grad_norm": 0.8246955871582031 }, { "bits_per_byte": 1.9125255016832254, "learning_rate": 0.0002862827825554171, "loss_nats": 1.3256616592407227, "step": 1500, "unclipped_grad_norm": 0.9061107635498047 }, { "bits_per_byte": 1.973825621895558, "learning_rate": 0.00028603665513185245, "loss_nats": 1.3681516647338867, "step": 1510, "unclipped_grad_norm": 0.9889123439788818 }, { "bits_per_byte": 1.980869687309597, "learning_rate": 0.000285788459525803, "loss_nats": 1.3730342388153076, "step": 1520, "unclipped_grad_norm": 0.9045401811599731 }, { "bits_per_byte": 1.9378121108395843, "learning_rate": 0.00028553819997824707, "loss_nats": 1.343189001083374, "step": 1530, "unclipped_grad_norm": 0.8896906971931458 }, { "bits_per_byte": 2.017761685746178, "learning_rate": 0.00028528588076542964, "loss_nats": 1.3986058235168457, "step": 1540, "unclipped_grad_norm": 0.9600386619567871 }, { "bits_per_byte": 1.9800400430018348, "learning_rate": 0.00028503150619878996, "loss_nats": 1.3724591732025146, "step": 1550, "unclipped_grad_norm": 0.8903042078018188 }, { "bits_per_byte": 2.037820882229562, "learning_rate": 0.0002847750806248876, "loss_nats": 1.412509799003601, "step": 1560, "unclipped_grad_norm": 0.9084417819976807 }, { "bits_per_byte": 2.015308525214271, "learning_rate": 0.000284516608425328, "loss_nats": 1.3969054222106934, "step": 1570, "unclipped_grad_norm": 0.9122006893157959 }, { "bits_per_byte": 1.985575476602505, "learning_rate": 0.000284256094016688, "loss_nats": 1.376296043395996, "step": 1580, "unclipped_grad_norm": 0.8903481960296631 }, { "bits_per_byte": 1.996281568601366, "learning_rate": 0.0002839935418504399, "loss_nats": 1.3837169408798218, "step": 1590, "unclipped_grad_norm": 0.933029294013977 }, { "bits_per_byte": 2.002402775110875, "learning_rate": 0.000283728956412876, "loss_nats": 1.3879598379135132, "step": 1600, "unclipped_grad_norm": 0.9777830243110657 }, { "bits_per_byte": 2.0119596790366705, "learning_rate": 0.00028346234222503136, "loss_nats": 1.3945841789245605, "step": 1610, "unclipped_grad_norm": 0.9627588391304016 }, { "bits_per_byte": 2.0159194075901357, "learning_rate": 0.00028319370384260675, "loss_nats": 1.3973288536071777, "step": 1620, "unclipped_grad_norm": 0.8731178641319275 }, { "bits_per_byte": 1.8905791395719242, "learning_rate": 0.00028292304585589105, "loss_nats": 1.3104496002197266, "step": 1630, "unclipped_grad_norm": 0.8998029828071594 }, { "bits_per_byte": 2.0235616286638756, "learning_rate": 0.0002826503728896824, "loss_nats": 1.4026260375976562, "step": 1640, "unclipped_grad_norm": 0.9714035391807556 }, { "bits_per_byte": 1.9588359580116574, "learning_rate": 0.0002823756896032095, "loss_nats": 1.3577616214752197, "step": 1650, "unclipped_grad_norm": 0.9218164682388306 }, { "bits_per_byte": 1.936817019221692, "learning_rate": 0.0002820990006900518, "loss_nats": 1.3424992561340332, "step": 1660, "unclipped_grad_norm": 0.8974215984344482 }, { "bits_per_byte": 2.096671109580354, "learning_rate": 0.00028182031087805947, "loss_nats": 1.4533016681671143, "step": 1670, "unclipped_grad_norm": 0.9378093481063843 }, { "bits_per_byte": 2.0150522710644854, "learning_rate": 0.0002815396249292725, "loss_nats": 1.3967278003692627, "step": 1680, "unclipped_grad_norm": 0.8720370531082153 }, { "bits_per_byte": 1.8838673446393523, "learning_rate": 0.0002812569476398391, "loss_nats": 1.3057973384857178, "step": 1690, "unclipped_grad_norm": 0.8577803373336792 }, { "bits_per_byte": 1.927341291107104, "learning_rate": 0.00028097228383993446, "loss_nats": 1.3359311819076538, "step": 1700, "unclipped_grad_norm": 0.9562329053878784 }, { "bits_per_byte": 1.9962937793695772, "learning_rate": 0.0002806856383936774, "loss_nats": 1.3837254047393799, "step": 1710, "unclipped_grad_norm": 0.9462636113166809 }, { "bits_per_byte": 1.995081645646296, "learning_rate": 0.00028039701619904774, "loss_nats": 1.382885217666626, "step": 1720, "unclipped_grad_norm": 0.8408851027488708 }, { "bits_per_byte": 1.894888164889292, "learning_rate": 0.0002801064221878024, "loss_nats": 1.3134363889694214, "step": 1730, "unclipped_grad_norm": 0.8187386989593506 }, { "bits_per_byte": 1.9475287866480313, "learning_rate": 0.00027981386132539134, "loss_nats": 1.349924087524414, "step": 1740, "unclipped_grad_norm": 0.9181671142578125 }, { "bits_per_byte": 2.0484275682562236, "learning_rate": 0.0002795193386108724, "loss_nats": 1.4198617935180664, "step": 1750, "unclipped_grad_norm": 0.9259082674980164 }, { "bits_per_byte": 1.9089950418263137, "learning_rate": 0.00027922285907682625, "loss_nats": 1.3232145309448242, "step": 1760, "unclipped_grad_norm": 0.9000674486160278 }, { "bits_per_byte": 1.9653874651136254, "learning_rate": 0.00027892442778927005, "loss_nats": 1.3623027801513672, "step": 1770, "unclipped_grad_norm": 0.8686579465866089 }, { "bits_per_byte": 1.9032847018697159, "learning_rate": 0.00027862404984757116, "loss_nats": 1.3192564249038696, "step": 1780, "unclipped_grad_norm": 0.8219738006591797 }, { "bits_per_byte": 1.8754542973244455, "learning_rate": 0.0002783217303843597, "loss_nats": 1.2999658584594727, "step": 1790, "unclipped_grad_norm": 0.8401536345481873 }, { "bits_per_byte": 1.9695277754504965, "learning_rate": 0.00027801747456544134, "loss_nats": 1.3651726245880127, "step": 1800, "unclipped_grad_norm": 0.9855192303657532 }, { "bits_per_byte": 1.952977025044714, "learning_rate": 0.00027771128758970843, "loss_nats": 1.3537005186080933, "step": 1810, "unclipped_grad_norm": 0.9152796268463135 }, { "bits_per_byte": 2.0236675699768067, "learning_rate": 0.00027740317468905165, "loss_nats": 1.4026994705200195, "step": 1820, "unclipped_grad_norm": 0.9160932898521423 }, { "bits_per_byte": 1.863515605666851, "learning_rate": 0.0002770931411282704, "loss_nats": 1.2916905879974365, "step": 1830, "unclipped_grad_norm": 0.8749093413352966 }, { "bits_per_byte": 1.9532654399502112, "learning_rate": 0.0002767811922049827, "loss_nats": 1.35390043258667, "step": 1840, "unclipped_grad_norm": 0.8691527843475342 }, { "bits_per_byte": 1.8425976998077092, "learning_rate": 0.00027646733324953503, "loss_nats": 1.277191400527954, "step": 1850, "unclipped_grad_norm": 0.918012797832489 }, { "bits_per_byte": 1.9936586611930571, "learning_rate": 0.00027615156962491097, "loss_nats": 1.3818988800048828, "step": 1860, "unclipped_grad_norm": 0.9178117513656616 }, { "bits_per_byte": 1.995530520365048, "learning_rate": 0.0002758339067266396, "loss_nats": 1.3831963539123535, "step": 1870, "unclipped_grad_norm": 0.8769021034240723 }, { "bits_per_byte": 1.8636951555543517, "learning_rate": 0.0002755143499827035, "loss_nats": 1.2918150424957275, "step": 1880, "unclipped_grad_norm": 0.8614026308059692 }, { "bits_per_byte": 1.9426209177403593, "learning_rate": 0.0002751929048534456, "loss_nats": 1.3465222120285034, "step": 1890, "unclipped_grad_norm": 0.8975904583930969 }, { "bits_per_byte": 1.861964838104021, "learning_rate": 0.0002748695768314763, "loss_nats": 1.2906156778335571, "step": 1900, "unclipped_grad_norm": 0.8269307613372803 }, { "bits_per_byte": 1.8502344174366214, "learning_rate": 0.00027454437144157947, "loss_nats": 1.282484769821167, "step": 1910, "unclipped_grad_norm": 0.8237605690956116 }, { "bits_per_byte": 1.9149965484108225, "learning_rate": 0.00027421729424061787, "loss_nats": 1.3273744583129883, "step": 1920, "unclipped_grad_norm": 0.8641052842140198 }, { "bits_per_byte": 1.8801597426520518, "learning_rate": 0.0002738883508174385, "loss_nats": 1.303227424621582, "step": 1930, "unclipped_grad_norm": 0.8907292485237122 }, { "bits_per_byte": 1.8629336163763313, "learning_rate": 0.0002735575467927768, "loss_nats": 1.2912871837615967, "step": 1940, "unclipped_grad_norm": 0.8837904930114746 }, { "bits_per_byte": 1.923144226495448, "learning_rate": 0.0002732248878191609, "loss_nats": 1.3330219984054565, "step": 1950, "unclipped_grad_norm": 1.0997384786605835 }, { "bits_per_byte": 1.88083511852199, "learning_rate": 0.0002728903795808148, "loss_nats": 1.303695559501648, "step": 1960, "unclipped_grad_norm": 0.8385953307151794 }, { "bits_per_byte": 1.8359603733629608, "learning_rate": 0.0002725540277935614, "loss_nats": 1.2725907564163208, "step": 1970, "unclipped_grad_norm": 0.839948832988739 }, { "bits_per_byte": 1.7988822896981462, "learning_rate": 0.00027221583820472456, "loss_nats": 1.2468901872634888, "step": 1980, "unclipped_grad_norm": 0.7658324837684631 }, { "bits_per_byte": 1.9198664091526536, "learning_rate": 0.0002718758165930314, "loss_nats": 1.3307499885559082, "step": 1990, "unclipped_grad_norm": 0.8840731382369995 }, { "bits_per_byte": 1.863330208369221, "learning_rate": 0.0002715339687685131, "loss_nats": 1.2915620803833008, "step": 2000, "unclipped_grad_norm": 0.8192090392112732 }, { "bits_per_byte": 1.9331472534175815, "learning_rate": 0.00027119030057240576, "loss_nats": 1.3399555683135986, "step": 2010, "unclipped_grad_norm": 0.8343153595924377 }, { "bits_per_byte": 1.9117808168049895, "learning_rate": 0.0002708448178770506, "loss_nats": 1.3251454830169678, "step": 2020, "unclipped_grad_norm": 0.913158118724823 }, { "bits_per_byte": 1.8970498148279866, "learning_rate": 0.00027049752658579376, "loss_nats": 1.3149347305297852, "step": 2030, "unclipped_grad_norm": 0.8877862691879272 }, { "bits_per_byte": 1.9115138997308505, "learning_rate": 0.00027014843263288514, "loss_nats": 1.324960470199585, "step": 2040, "unclipped_grad_norm": 0.8749900460243225 }, { "bits_per_byte": 1.8657040848990802, "learning_rate": 0.00026979754198337724, "loss_nats": 1.2932075262069702, "step": 2050, "unclipped_grad_norm": 0.9185486435890198 }, { "bits_per_byte": 1.7675556498434857, "learning_rate": 0.00026944486063302305, "loss_nats": 1.225176215171814, "step": 2060, "unclipped_grad_norm": 0.7904725670814514 }, { "bits_per_byte": 1.8617049723185672, "learning_rate": 0.0002690903946081738, "loss_nats": 1.290435552597046, "step": 2070, "unclipped_grad_norm": 0.8348630666732788 }, { "bits_per_byte": 1.973646415973359, "learning_rate": 0.00026873414996567596, "loss_nats": 1.3680274486541748, "step": 2080, "unclipped_grad_norm": 0.8117388486862183 }, { "bits_per_byte": 1.8279640400110946, "learning_rate": 0.00026837613279276736, "loss_nats": 1.2670481204986572, "step": 2090, "unclipped_grad_norm": 0.7758682370185852 }, { "bits_per_byte": 1.8962239541385433, "learning_rate": 0.0002680163492069739, "loss_nats": 1.3143622875213623, "step": 2100, "unclipped_grad_norm": 0.8137099146842957 }, { "bits_per_byte": 1.8350517890184526, "learning_rate": 0.00026765480535600414, "loss_nats": 1.271960973739624, "step": 2110, "unclipped_grad_norm": 0.9251447319984436 }, { "bits_per_byte": 1.870564486595114, "learning_rate": 0.0002672915074176452, "loss_nats": 1.2965764999389648, "step": 2120, "unclipped_grad_norm": 0.9138389825820923 }, { "bits_per_byte": 1.8882444750864615, "learning_rate": 0.00026692646159965624, "loss_nats": 1.3088313341140747, "step": 2130, "unclipped_grad_norm": 0.8262670040130615 }, { "bits_per_byte": 1.7726164113277747, "learning_rate": 0.0002665596741396633, "loss_nats": 1.2286840677261353, "step": 2140, "unclipped_grad_norm": 0.7359873056411743 }, { "bits_per_byte": 1.8268258588276844, "learning_rate": 0.000266191151305052, "loss_nats": 1.2662591934204102, "step": 2150, "unclipped_grad_norm": 0.8173519372940063 }, { "bits_per_byte": 1.8637190611428216, "learning_rate": 0.00026582089939286075, "loss_nats": 1.291831612586975, "step": 2160, "unclipped_grad_norm": 0.8174240589141846 }, { "bits_per_byte": 1.8528119214251022, "learning_rate": 0.0002654489247296732, "loss_nats": 1.2842713594436646, "step": 2170, "unclipped_grad_norm": 0.8957990407943726 }, { "bits_per_byte": 1.782856774308099, "learning_rate": 0.00026507523367151007, "loss_nats": 1.2357821464538574, "step": 2180, "unclipped_grad_norm": 0.8048707246780396 }, { "bits_per_byte": 1.8923490130319534, "learning_rate": 0.00026469983260372037, "loss_nats": 1.3116763830184937, "step": 2190, "unclipped_grad_norm": 0.8952655792236328 }, { "bits_per_byte": 1.9442205283760343, "learning_rate": 0.0002643227279408727, "loss_nats": 1.3476309776306152, "step": 2200, "unclipped_grad_norm": 0.8895230889320374 }, { "bits_per_byte": 1.9876000563684622, "learning_rate": 0.0002639439261266453, "loss_nats": 1.377699375152588, "step": 2210, "unclipped_grad_norm": 0.8686981797218323 }, { "bits_per_byte": 1.852814845130167, "learning_rate": 0.000263563433633716, "loss_nats": 1.284273386001587, "step": 2220, "unclipped_grad_norm": 0.8053512573242188 }, { "bits_per_byte": 1.859576858996791, "learning_rate": 0.00026318125696365156, "loss_nats": 1.2889604568481445, "step": 2230, "unclipped_grad_norm": 0.8349927663803101 }, { "bits_per_byte": 1.8206912376714082, "learning_rate": 0.000262797402646797, "loss_nats": 1.2620069980621338, "step": 2240, "unclipped_grad_norm": 0.8471757769584656 }, { "bits_per_byte": 1.8270817690121683, "learning_rate": 0.00026241187724216334, "loss_nats": 1.2664365768432617, "step": 2250, "unclipped_grad_norm": 0.8009039163589478 }, { "bits_per_byte": 1.8453999851208678, "learning_rate": 0.0002620246873373162, "loss_nats": 1.2791337966918945, "step": 2260, "unclipped_grad_norm": 0.8003636002540588 }, { "bits_per_byte": 1.9009572606555896, "learning_rate": 0.00026163583954826264, "loss_nats": 1.317643165588379, "step": 2270, "unclipped_grad_norm": 0.8468067646026611 }, { "bits_per_byte": 1.7920475271702074, "learning_rate": 0.0002612453405193386, "loss_nats": 1.2421526908874512, "step": 2280, "unclipped_grad_norm": 0.9071624875068665 }, { "bits_per_byte": 1.8281408381761814, "learning_rate": 0.000260853196923095, "loss_nats": 1.2671706676483154, "step": 2290, "unclipped_grad_norm": 0.822325587272644 }, { "bits_per_byte": 1.8827026781277094, "learning_rate": 0.0002604594154601839, "loss_nats": 1.3049900531768799, "step": 2300, "unclipped_grad_norm": 0.8915430307388306 }, { "bits_per_byte": 1.831042185496237, "learning_rate": 0.0002600640028592441, "loss_nats": 1.269181728363037, "step": 2310, "unclipped_grad_norm": 0.8500391840934753 }, { "bits_per_byte": 1.8431096921593277, "learning_rate": 0.0002596669658767859, "loss_nats": 1.2775462865829468, "step": 2320, "unclipped_grad_norm": 0.8417274355888367 }, { "bits_per_byte": 1.7906957435344257, "learning_rate": 0.000259268311297076, "loss_nats": 1.241215705871582, "step": 2330, "unclipped_grad_norm": 0.777140736579895 }, { "bits_per_byte": 1.825728265549878, "learning_rate": 0.00025886804593202114, "loss_nats": 1.265498399734497, "step": 2340, "unclipped_grad_norm": 0.8567083477973938 }, { "bits_per_byte": 1.7348746466132805, "learning_rate": 0.00025846617662105193, "loss_nats": 1.2025234699249268, "step": 2350, "unclipped_grad_norm": 0.7593752145767212 }, { "bits_per_byte": 1.8440781264663364, "learning_rate": 0.00025806271023100637, "loss_nats": 1.2782175540924072, "step": 2360, "unclipped_grad_norm": 0.8551052212715149 }, { "bits_per_byte": 1.871989878805465, "learning_rate": 0.00025765765365601165, "loss_nats": 1.2975645065307617, "step": 2370, "unclipped_grad_norm": 0.8494746685028076 }, { "bits_per_byte": 1.7757177744707837, "learning_rate": 0.00025725101381736707, "loss_nats": 1.2308337688446045, "step": 2380, "unclipped_grad_norm": 0.7841326594352722 }, { "bits_per_byte": 1.7735628318554726, "learning_rate": 0.00025684279766342567, "loss_nats": 1.2293400764465332, "step": 2390, "unclipped_grad_norm": 0.8951802849769592 }, { "bits_per_byte": 1.8680005692360513, "learning_rate": 0.00025643301216947504, "loss_nats": 1.2947993278503418, "step": 2400, "unclipped_grad_norm": 0.8339796662330627 }, { "bits_per_byte": 1.8459044102358484, "learning_rate": 0.00025602166433761866, "loss_nats": 1.279483437538147, "step": 2410, "unclipped_grad_norm": 0.9356110692024231 }, { "bits_per_byte": 1.931278833898608, "learning_rate": 0.0002556087611966561, "loss_nats": 1.338660478591919, "step": 2420, "unclipped_grad_norm": 0.8573734760284424 }, { "bits_per_byte": 1.825236739133712, "learning_rate": 0.00025519430980196277, "loss_nats": 1.265157699584961, "step": 2430, "unclipped_grad_norm": 0.8783476948738098 }, { "bits_per_byte": 1.7732133631089195, "learning_rate": 0.0002547783172353695, "loss_nats": 1.229097843170166, "step": 2440, "unclipped_grad_norm": 0.8393950462341309 }, { "bits_per_byte": 1.8696945983470499, "learning_rate": 0.0002543607906050415, "loss_nats": 1.295973539352417, "step": 2450, "unclipped_grad_norm": 0.8328628540039062 }, { "bits_per_byte": 1.8721511985319743, "learning_rate": 0.0002539417370453568, "loss_nats": 1.2976763248443604, "step": 2460, "unclipped_grad_norm": 0.8645251989364624 }, { "bits_per_byte": 1.8203225068679583, "learning_rate": 0.00025352116371678464, "loss_nats": 1.261751413345337, "step": 2470, "unclipped_grad_norm": 0.8000954985618591 }, { "bits_per_byte": 1.8206716316492098, "learning_rate": 0.0002530990778057627, "loss_nats": 1.261993408203125, "step": 2480, "unclipped_grad_norm": 0.7867690920829773 }, { "bits_per_byte": 1.7380601092725614, "learning_rate": 0.00025267548652457475, "loss_nats": 1.2047314643859863, "step": 2490, "unclipped_grad_norm": 0.7984139919281006 }, { "bits_per_byte": 1.822108202731867, "learning_rate": 0.00025225039711122696, "loss_nats": 1.2629891633987427, "step": 2500, "unclipped_grad_norm": 0.7798662781715393 }, { "bits_per_byte": 1.7878630172920644, "learning_rate": 0.00025182381682932464, "loss_nats": 1.2392522096633911, "step": 2510, "unclipped_grad_norm": 0.8921790719032288 }, { "bits_per_byte": 1.7912486677569497, "learning_rate": 0.0002513957529679477, "loss_nats": 1.2415989637374878, "step": 2520, "unclipped_grad_norm": 0.8250295519828796 }, { "bits_per_byte": 1.7871405181757898, "learning_rate": 0.00025096621284152656, "loss_nats": 1.2387514114379883, "step": 2530, "unclipped_grad_norm": 0.7922425270080566 }, { "bits_per_byte": 1.7956409326773348, "learning_rate": 0.0002505352037897168, "loss_nats": 1.2446434497833252, "step": 2540, "unclipped_grad_norm": 0.8244069218635559 }, { "bits_per_byte": 1.833951787983534, "learning_rate": 0.0002501027331772739, "loss_nats": 1.2711985111236572, "step": 2550, "unclipped_grad_norm": 0.8060020804405212 }, { "bits_per_byte": 1.7851159384098327, "learning_rate": 0.00024966880839392746, "loss_nats": 1.2373480796813965, "step": 2560, "unclipped_grad_norm": 0.7821792960166931 }, { "bits_per_byte": 1.768823849910948, "learning_rate": 0.0002492334368542548, "loss_nats": 1.2260552644729614, "step": 2570, "unclipped_grad_norm": 0.7924907207489014 }, { "bits_per_byte": 1.8169237857216074, "learning_rate": 0.00024879662599755416, "loss_nats": 1.2593955993652344, "step": 2580, "unclipped_grad_norm": 0.813592255115509 }, { "bits_per_byte": 1.7917042497990852, "learning_rate": 0.00024835838328771803, "loss_nats": 1.2419147491455078, "step": 2590, "unclipped_grad_norm": 0.8239418268203735 }, { "bits_per_byte": 1.8340940176357976, "learning_rate": 0.0002479187162131051, "loss_nats": 1.2712970972061157, "step": 2600, "unclipped_grad_norm": 0.7961696982383728 }, { "bits_per_byte": 1.7288738279593754, "learning_rate": 0.00024747763228641264, "loss_nats": 1.198364019393921, "step": 2610, "unclipped_grad_norm": 0.7893867492675781 }, { "bits_per_byte": 1.7710386424887592, "learning_rate": 0.00024703513904454795, "loss_nats": 1.2275904417037964, "step": 2620, "unclipped_grad_norm": 0.7791327238082886 }, { "bits_per_byte": 1.783225161146247, "learning_rate": 0.00024659124404849956, "loss_nats": 1.2360374927520752, "step": 2630, "unclipped_grad_norm": 0.7614359259605408 }, { "bits_per_byte": 1.8200944578729141, "learning_rate": 0.00024614595488320833, "loss_nats": 1.2615933418273926, "step": 2640, "unclipped_grad_norm": 0.831463098526001 }, { "bits_per_byte": 1.7566091260987187, "learning_rate": 0.00024569927915743745, "loss_nats": 1.2175886631011963, "step": 2650, "unclipped_grad_norm": 0.7739477157592773 }, { "bits_per_byte": 1.7470520501902724, "learning_rate": 0.0002452512245036427, "loss_nats": 1.2109642028808594, "step": 2660, "unclipped_grad_norm": 0.7929992079734802 }, { "bits_per_byte": 1.775358846678433, "learning_rate": 0.00024480179857784177, "loss_nats": 1.230584979057312, "step": 2670, "unclipped_grad_norm": 0.7438913583755493 }, { "bits_per_byte": 1.7678870604116983, "learning_rate": 0.00024435100905948387, "loss_nats": 1.2254059314727783, "step": 2680, "unclipped_grad_norm": 0.8285056352615356 }, { "bits_per_byte": 1.8436208245976922, "learning_rate": 0.00024389886365131793, "loss_nats": 1.2779005765914917, "step": 2690, "unclipped_grad_norm": 0.7684076428413391 }, { "bits_per_byte": 1.825501592416041, "learning_rate": 0.00024344537007926163, "loss_nats": 1.2653412818908691, "step": 2700, "unclipped_grad_norm": 0.8987715840339661 }, { "bits_per_byte": 1.7152053347994027, "learning_rate": 0.00024299053609226878, "loss_nats": 1.188889741897583, "step": 2710, "unclipped_grad_norm": 0.8146651983261108 }, { "bits_per_byte": 1.8198911743795942, "learning_rate": 0.00024253436946219733, "loss_nats": 1.2614524364471436, "step": 2720, "unclipped_grad_norm": 0.8013731241226196 }, { "bits_per_byte": 1.7448410412307802, "learning_rate": 0.0002420768779836764, "loss_nats": 1.2094316482543945, "step": 2730, "unclipped_grad_norm": 0.7481213212013245 }, { "bits_per_byte": 1.7681415947349752, "learning_rate": 0.00024161806947397305, "loss_nats": 1.2255823612213135, "step": 2740, "unclipped_grad_norm": 0.8155582547187805 }, { "bits_per_byte": 1.781660806953999, "learning_rate": 0.00024115795177285886, "loss_nats": 1.2349531650543213, "step": 2750, "unclipped_grad_norm": 0.816902756690979 }, { "bits_per_byte": 1.869134278870539, "learning_rate": 0.0002406965327424758, "loss_nats": 1.2955851554870605, "step": 2760, "unclipped_grad_norm": 0.8422117233276367 }, { "bits_per_byte": 1.7766440730283308, "learning_rate": 0.00024023382026720197, "loss_nats": 1.231475830078125, "step": 2770, "unclipped_grad_norm": 0.8072130084037781 }, { "bits_per_byte": 1.778421169759696, "learning_rate": 0.00023976982225351698, "loss_nats": 1.2327076196670532, "step": 2780, "unclipped_grad_norm": 0.8882554173469543 }, { "bits_per_byte": 1.7671284449387423, "learning_rate": 0.0002393045466298666, "loss_nats": 1.2248800992965698, "step": 2790, "unclipped_grad_norm": 0.8406802415847778 }, { "bits_per_byte": 1.719355792112675, "learning_rate": 0.00023883800134652763, "loss_nats": 1.191766619682312, "step": 2800, "unclipped_grad_norm": 0.8179776072502136 }, { "bits_per_byte": 1.7902867687906738, "learning_rate": 0.0002383701943754715, "loss_nats": 1.2409322261810303, "step": 2810, "unclipped_grad_norm": 0.8046079874038696 }, { "bits_per_byte": 1.7916516231079211, "learning_rate": 0.00023790113371022884, "loss_nats": 1.2418782711029053, "step": 2820, "unclipped_grad_norm": 0.7889978885650635 }, { "bits_per_byte": 1.717342391219024, "learning_rate": 0.0002374308273657521, "loss_nats": 1.190371036529541, "step": 2830, "unclipped_grad_norm": 0.7270697355270386 }, { "bits_per_byte": 1.8021411889493455, "learning_rate": 0.00023695928337827911, "loss_nats": 1.2491490840911865, "step": 2840, "unclipped_grad_norm": 0.8529788255691528 }, { "bits_per_byte": 1.7264863648000979, "learning_rate": 0.00023648650980519564, "loss_nats": 1.196709156036377, "step": 2850, "unclipped_grad_norm": 0.7487845420837402 }, { "bits_per_byte": 1.698717702026487, "learning_rate": 0.00023601251472489757, "loss_nats": 1.1774613857269287, "step": 2860, "unclipped_grad_norm": 0.8107967972755432 }, { "bits_per_byte": 1.8323354950307253, "learning_rate": 0.00023553730623665305, "loss_nats": 1.270078182220459, "step": 2870, "unclipped_grad_norm": 0.822874128818512 }, { "bits_per_byte": 1.7418315168233318, "learning_rate": 0.00023506089246046403, "loss_nats": 1.2073456048965454, "step": 2880, "unclipped_grad_norm": 0.7772940993309021 }, { "bits_per_byte": 1.7973201712803593, "learning_rate": 0.00023458328153692736, "loss_nats": 1.245807409286499, "step": 2890, "unclipped_grad_norm": 0.7877682447433472 }, { "bits_per_byte": 1.7076477291898875, "learning_rate": 0.0002341044816270961, "loss_nats": 1.1836512088775635, "step": 2900, "unclipped_grad_norm": 0.8104777932167053 }, { "bits_per_byte": 1.7337485042154308, "learning_rate": 0.00023362450091233953, "loss_nats": 1.2017428874969482, "step": 2910, "unclipped_grad_norm": 0.8031578063964844 }, { "bits_per_byte": 1.8229534974608574, "learning_rate": 0.00023314334759420388, "loss_nats": 1.2635750770568848, "step": 2920, "unclipped_grad_norm": 0.7966766953468323 }, { "bits_per_byte": 1.766033431400699, "learning_rate": 0.0002326610298942717, "loss_nats": 1.22412109375, "step": 2930, "unclipped_grad_norm": 0.8636581301689148 }, { "bits_per_byte": 1.8135838826418518, "learning_rate": 0.00023217755605402175, "loss_nats": 1.2570805549621582, "step": 2940, "unclipped_grad_norm": 0.8673250675201416 }, { "bits_per_byte": 1.7613176671140407, "learning_rate": 0.00023169293433468802, "loss_nats": 1.2208523750305176, "step": 2950, "unclipped_grad_norm": 0.7721947431564331 }, { "bits_per_byte": 1.7305283010606756, "learning_rate": 0.00023120717301711857, "loss_nats": 1.1995108127593994, "step": 2960, "unclipped_grad_norm": 0.7505601644515991 }, { "bits_per_byte": 1.8081494028572378, "learning_rate": 0.00023072028040163408, "loss_nats": 1.253313660621643, "step": 2970, "unclipped_grad_norm": 0.7939547896385193 }, { "bits_per_byte": 1.72656255311443, "learning_rate": 0.00023023226480788593, "loss_nats": 1.196761965751648, "step": 2980, "unclipped_grad_norm": 0.7610272765159607 }, { "bits_per_byte": 1.8584056571444152, "learning_rate": 0.00022974313457471416, "loss_nats": 1.2881486415863037, "step": 2990, "unclipped_grad_norm": 0.833854615688324 }, { "bits_per_byte": 1.7959412143857414, "learning_rate": 0.0002292528980600049, "loss_nats": 1.2448515892028809, "step": 3000, "unclipped_grad_norm": 0.7871666550636292 }, { "bits_per_byte": 1.7543190051198296, "learning_rate": 0.00022876156364054758, "loss_nats": 1.216001272201538, "step": 3010, "unclipped_grad_norm": 0.7730440497398376 }, { "bits_per_byte": 1.7304921847039942, "learning_rate": 0.0002282691397118918, "loss_nats": 1.1994857788085938, "step": 3020, "unclipped_grad_norm": 0.8199000954627991 }, { "bits_per_byte": 1.7578830015936595, "learning_rate": 0.00022777563468820376, "loss_nats": 1.218471646308899, "step": 3030, "unclipped_grad_norm": 0.9139093160629272 }, { "bits_per_byte": 1.7370483353375354, "learning_rate": 0.00022728105700212282, "loss_nats": 1.204030156135559, "step": 3040, "unclipped_grad_norm": 0.7832452058792114 }, { "bits_per_byte": 1.745498014957076, "learning_rate": 0.00022678541510461698, "loss_nats": 1.2098870277404785, "step": 3050, "unclipped_grad_norm": 0.7990915179252625 }, { "bits_per_byte": 1.7370017280391516, "learning_rate": 0.00022628871746483876, "loss_nats": 1.2039978504180908, "step": 3060, "unclipped_grad_norm": 0.8140543103218079 }, { "bits_per_byte": 1.7324813360438736, "learning_rate": 0.0002257909725699804, "loss_nats": 1.200864553451538, "step": 3070, "unclipped_grad_norm": 0.8290132284164429 }, { "bits_per_byte": 1.73554830265671, "learning_rate": 0.00022529218892512886, "loss_nats": 1.2029904127120972, "step": 3080, "unclipped_grad_norm": 0.8363073468208313 }, { "bits_per_byte": 1.7064868462965634, "learning_rate": 0.00022479237505312034, "loss_nats": 1.1828465461730957, "step": 3090, "unclipped_grad_norm": 0.7568578124046326 }, { "bits_per_byte": 1.8015361539836103, "learning_rate": 0.00022429153949439496, "loss_nats": 1.2487297058105469, "step": 3100, "unclipped_grad_norm": 0.8538148999214172 }, { "bits_per_byte": 1.7730212584879057, "learning_rate": 0.00022378969080685053, "loss_nats": 1.2289646863937378, "step": 3110, "unclipped_grad_norm": 0.8123647570610046 }, { "bits_per_byte": 1.664611306585931, "learning_rate": 0.0002232868375656965, "loss_nats": 1.1538206338882446, "step": 3120, "unclipped_grad_norm": 0.814990758895874 }, { "bits_per_byte": 1.7603762340832176, "learning_rate": 0.00022278298836330733, "loss_nats": 1.2201998233795166, "step": 3130, "unclipped_grad_norm": 0.8308927416801453 }, { "bits_per_byte": 1.713619138810495, "learning_rate": 0.00022227815180907572, "loss_nats": 1.1877902746200562, "step": 3140, "unclipped_grad_norm": 0.7710813879966736 }, { "bits_per_byte": 1.7093383186478688, "learning_rate": 0.00022177233652926546, "loss_nats": 1.1848230361938477, "step": 3150, "unclipped_grad_norm": 0.7859835624694824 }, { "bits_per_byte": 1.7563532159142348, "learning_rate": 0.00022126555116686416, "loss_nats": 1.2174112796783447, "step": 3160, "unclipped_grad_norm": 0.811743974685669 }, { "bits_per_byte": 1.8161149513145995, "learning_rate": 0.00022075780438143532, "loss_nats": 1.258834958076477, "step": 3170, "unclipped_grad_norm": 0.8695415258407593 }, { "bits_per_byte": 1.6889931150161004, "learning_rate": 0.00022024910484897065, "loss_nats": 1.1707208156585693, "step": 3180, "unclipped_grad_norm": 0.8201394081115723 }, { "bits_per_byte": 1.7704115937437135, "learning_rate": 0.00021973946126174158, "loss_nats": 1.2271558046340942, "step": 3190, "unclipped_grad_norm": 0.8031932711601257 }, { "bits_per_byte": 1.6638831320421776, "learning_rate": 0.00021922888232815086, "loss_nats": 1.1533159017562866, "step": 3200, "unclipped_grad_norm": 0.7531365156173706 }, { "bits_per_byte": 1.7080410535124106, "learning_rate": 0.0002187173767725837, "loss_nats": 1.1839238405227661, "step": 3210, "unclipped_grad_norm": 0.7989531755447388 }, { "bits_per_byte": 1.7142219380017691, "learning_rate": 0.0002182049533352588, "loss_nats": 1.1882081031799316, "step": 3220, "unclipped_grad_norm": 0.8516196012496948 }, { "bits_per_byte": 1.687988048404458, "learning_rate": 0.00021769162077207884, "loss_nats": 1.1700241565704346, "step": 3230, "unclipped_grad_norm": 0.8453141450881958 }, { "bits_per_byte": 1.725164850110868, "learning_rate": 0.00021717738785448102, "loss_nats": 1.1957931518554688, "step": 3240, "unclipped_grad_norm": 0.8131914734840393 }, { "bits_per_byte": 1.7438406181507111, "learning_rate": 0.00021666226336928708, "loss_nats": 1.2087382078170776, "step": 3250, "unclipped_grad_norm": 0.8390238285064697 }, { "bits_per_byte": 1.8351064795014271, "learning_rate": 0.0002161462561185532, "loss_nats": 1.2719988822937012, "step": 3260, "unclipped_grad_norm": 0.8244672417640686 }, { "bits_per_byte": 1.6914478233918653, "learning_rate": 0.00021562937491941957, "loss_nats": 1.1724222898483276, "step": 3270, "unclipped_grad_norm": 0.8076629042625427 }, { "bits_per_byte": 1.6199020087370317, "learning_rate": 0.00021511162860395975, "loss_nats": 1.1228305101394653, "step": 3280, "unclipped_grad_norm": 0.7831076979637146 }, { "bits_per_byte": 1.6792207168287738, "learning_rate": 0.0002145930260190298, "loss_nats": 1.1639471054077148, "step": 3290, "unclipped_grad_norm": 0.8022177219390869 }, { "bits_per_byte": 1.7567814527148833, "learning_rate": 0.00021407357602611698, "loss_nats": 1.2177081108093262, "step": 3300, "unclipped_grad_norm": 0.8066262602806091 }, { "bits_per_byte": 1.745768887632185, "learning_rate": 0.0002135532875011885, "loss_nats": 1.210074782371521, "step": 3310, "unclipped_grad_norm": 0.8170367479324341 }, { "bits_per_byte": 1.7488066171943748, "learning_rate": 0.0002130321693345397, "loss_nats": 1.2121803760528564, "step": 3320, "unclipped_grad_norm": 0.8205949664115906 }, { "bits_per_byte": 1.6972405430382262, "learning_rate": 0.00021251023043064224, "loss_nats": 1.176437497138977, "step": 3330, "unclipped_grad_norm": 0.7750418782234192 }, { "bits_per_byte": 1.7083963696690934, "learning_rate": 0.00021198747970799188, "loss_nats": 1.184170126914978, "step": 3340, "unclipped_grad_norm": 0.7907351851463318 }, { "bits_per_byte": 1.7221589373390878, "learning_rate": 0.00021146392609895618, "loss_nats": 1.1937096118927002, "step": 3350, "unclipped_grad_norm": 0.7935147285461426 }, { "bits_per_byte": 1.6939794080125659, "learning_rate": 0.00021093957854962178, "loss_nats": 1.1741770505905151, "step": 3360, "unclipped_grad_norm": 0.7715696096420288 }, { "bits_per_byte": 1.6536716621471985, "learning_rate": 0.00021041444601964157, "loss_nats": 1.146237850189209, "step": 3370, "unclipped_grad_norm": 0.7847044467926025 }, { "bits_per_byte": 1.7430156173745224, "learning_rate": 0.00020988853748208164, "loss_nats": 1.2081663608551025, "step": 3380, "unclipped_grad_norm": 0.9182605743408203 }, { "bits_per_byte": 1.7282608817917002, "learning_rate": 0.00020936186192326782, "loss_nats": 1.197939157485962, "step": 3390, "unclipped_grad_norm": 0.8387140035629272 }, { "bits_per_byte": 1.7168453613580306, "learning_rate": 0.00020883442834263225, "loss_nats": 1.1900265216827393, "step": 3400, "unclipped_grad_norm": 0.827458918094635 }, { "bits_per_byte": 1.744452876387783, "learning_rate": 0.0002083062457525596, "loss_nats": 1.2091625928878784, "step": 3410, "unclipped_grad_norm": 0.8461803197860718 }, { "bits_per_byte": 1.7169175940713928, "learning_rate": 0.00020777732317823306, "loss_nats": 1.1900765895843506, "step": 3420, "unclipped_grad_norm": 0.7754979133605957 }, { "bits_per_byte": 1.665198971303929, "learning_rate": 0.00020724766965748005, "loss_nats": 1.1542279720306396, "step": 3430, "unclipped_grad_norm": 0.784263551235199 }, { "bits_per_byte": 1.6807204055442975, "learning_rate": 0.00020671729424061788, "loss_nats": 1.1649866104125977, "step": 3440, "unclipped_grad_norm": 0.768701434135437 }, { "bits_per_byte": 1.674807642007634, "learning_rate": 0.00020618620599029906, "loss_nats": 1.1608881950378418, "step": 3450, "unclipped_grad_norm": 0.8573304414749146 }, { "bits_per_byte": 1.6607244986764313, "learning_rate": 0.00020565441398135649, "loss_nats": 1.151126503944397, "step": 3460, "unclipped_grad_norm": 0.7546505331993103 }, { "bits_per_byte": 1.7168016777647113, "learning_rate": 0.00020512192730064827, "loss_nats": 1.1899962425231934, "step": 3470, "unclipped_grad_norm": 0.8346678614616394 }, { "bits_per_byte": 1.7794081781929978, "learning_rate": 0.00020458875504690273, "loss_nats": 1.2333917617797852, "step": 3480, "unclipped_grad_norm": 0.8080501556396484 }, { "bits_per_byte": 1.7098234817059528, "learning_rate": 0.00020405490633056245, "loss_nats": 1.1851593255996704, "step": 3490, "unclipped_grad_norm": 0.7828541398048401 }, { "bits_per_byte": 1.6618220919542377, "learning_rate": 0.00020352039027362917, "loss_nats": 1.15188729763031, "step": 3500, "unclipped_grad_norm": 0.8354499936103821 }, { "bits_per_byte": 1.670962969847596, "learning_rate": 0.0002029852160095075, "loss_nats": 1.158223271369934, "step": 3510, "unclipped_grad_norm": 0.7484581470489502 }, { "bits_per_byte": 1.7077951183216769, "learning_rate": 0.000202449392682849, "loss_nats": 1.1837533712387085, "step": 3520, "unclipped_grad_norm": 0.8215554356575012 }, { "bits_per_byte": 1.6640239858332344, "learning_rate": 0.0002019129294493959, "loss_nats": 1.1534135341644287, "step": 3530, "unclipped_grad_norm": 0.7679988741874695 }, { "bits_per_byte": 1.6934656958344385, "learning_rate": 0.00020137583547582478, "loss_nats": 1.173820972442627, "step": 3540, "unclipped_grad_norm": 0.849611759185791 }, { "bits_per_byte": 1.7380399873024104, "learning_rate": 0.00020083811993958973, "loss_nats": 1.2047175168991089, "step": 3550, "unclipped_grad_norm": 0.8348926305770874 }, { "bits_per_byte": 1.6724150193688307, "learning_rate": 0.00020029979202876563, "loss_nats": 1.1592297554016113, "step": 3560, "unclipped_grad_norm": 0.82209312915802 }, { "bits_per_byte": 1.707797526078789, "learning_rate": 0.00019976086094189118, "loss_nats": 1.1837550401687622, "step": 3570, "unclipped_grad_norm": 0.814924955368042 }, { "bits_per_byte": 1.725196322935976, "learning_rate": 0.00019922133588781175, "loss_nats": 1.1958149671554565, "step": 3580, "unclipped_grad_norm": 0.8165888786315918 }, { "bits_per_byte": 1.7086591591596116, "learning_rate": 0.0001986812260855219, "loss_nats": 1.1843522787094116, "step": 3590, "unclipped_grad_norm": 0.8172020316123962 }, { "bits_per_byte": 1.7535423314685332, "learning_rate": 0.00019814054076400787, "loss_nats": 1.2154629230499268, "step": 3600, "unclipped_grad_norm": 0.8490144610404968 }, { "bits_per_byte": 1.7320164669385913, "learning_rate": 0.00019759928916209004, "loss_nats": 1.2005423307418823, "step": 3610, "unclipped_grad_norm": 0.8558303117752075 }, { "bits_per_byte": 1.671523289324107, "learning_rate": 0.00019705748052826493, "loss_nats": 1.1586116552352905, "step": 3620, "unclipped_grad_norm": 0.7700185179710388 }, { "bits_per_byte": 1.6783824733884685, "learning_rate": 0.00019651512412054723, "loss_nats": 1.1633660793304443, "step": 3630, "unclipped_grad_norm": 0.8705776333808899 }, { "bits_per_byte": 1.6085100499265301, "learning_rate": 0.00019597222920631144, "loss_nats": 1.1149342060089111, "step": 3640, "unclipped_grad_norm": 0.7363900542259216 }, { "bits_per_byte": 1.69346002040696, "learning_rate": 0.0001954288050621338, "loss_nats": 1.1738170385360718, "step": 3650, "unclipped_grad_norm": 0.7923496961593628 }, { "bits_per_byte": 1.6919579239343243, "learning_rate": 0.00019488486097363363, "loss_nats": 1.1727758646011353, "step": 3660, "unclipped_grad_norm": 0.8016297221183777 }, { "bits_per_byte": 1.7219477426438283, "learning_rate": 0.00019434040623531452, "loss_nats": 1.1935632228851318, "step": 3670, "unclipped_grad_norm": 0.8946071863174438 }, { "bits_per_byte": 1.6611216066172736, "learning_rate": 0.0001937954501504058, "loss_nats": 1.1514017581939697, "step": 3680, "unclipped_grad_norm": 0.7832621932029724 }, { "bits_per_byte": 1.7299588665036687, "learning_rate": 0.00019325000203070333, "loss_nats": 1.1991161108016968, "step": 3690, "unclipped_grad_norm": 0.8317174911499023 }, { "bits_per_byte": 1.6188276331170917, "learning_rate": 0.0001927040711964106, "loss_nats": 1.1220858097076416, "step": 3700, "unclipped_grad_norm": 0.7848891615867615 }, { "bits_per_byte": 1.69280218676741, "learning_rate": 0.00019215766697597923, "loss_nats": 1.17336106300354, "step": 3710, "unclipped_grad_norm": 0.7887690663337708 }, { "bits_per_byte": 1.7133850704226707, "learning_rate": 0.00019161079870594978, "loss_nats": 1.1876280307769775, "step": 3720, "unclipped_grad_norm": 0.8315525650978088 }, { "bits_per_byte": 1.6847540866376338, "learning_rate": 0.0001910634757307922, "loss_nats": 1.1677825450897217, "step": 3730, "unclipped_grad_norm": 0.8175208568572998 }, { "bits_per_byte": 1.6646517225088835, "learning_rate": 0.0001905157074027459, "loss_nats": 1.153848648071289, "step": 3740, "unclipped_grad_norm": 0.795343816280365 }, { "bits_per_byte": 1.6976003307438312, "learning_rate": 0.00018996750308166037, "loss_nats": 1.1766868829727173, "step": 3750, "unclipped_grad_norm": 0.8138338327407837 }, { "bits_per_byte": 1.7200370153927425, "learning_rate": 0.00018941887213483482, "loss_nats": 1.1922388076782227, "step": 3760, "unclipped_grad_norm": 0.8444334268569946 }, { "bits_per_byte": 1.7215936303657018, "learning_rate": 0.00018886982393685845, "loss_nats": 1.1933177709579468, "step": 3770, "unclipped_grad_norm": 0.7583279609680176 }, { "bits_per_byte": 1.7717356881727062, "learning_rate": 0.00018832036786945008, "loss_nats": 1.2280735969543457, "step": 3780, "unclipped_grad_norm": 0.8467698693275452 }, { "bits_per_byte": 1.6571842358084203, "learning_rate": 0.0001877705133212979, "loss_nats": 1.1486725807189941, "step": 3790, "unclipped_grad_norm": 0.769040584564209 }, { "bits_per_byte": 1.6766210270783317, "learning_rate": 0.00018722026968789906, "loss_nats": 1.1621451377868652, "step": 3800, "unclipped_grad_norm": 0.7948989272117615 }, { "bits_per_byte": 1.6280214816669156, "learning_rate": 0.00018666964637139907, "loss_nats": 1.1284584999084473, "step": 3810, "unclipped_grad_norm": 0.8209450840950012 }, { "bits_per_byte": 1.6241315780697008, "learning_rate": 0.00018611865278043115, "loss_nats": 1.1257622241973877, "step": 3820, "unclipped_grad_norm": 0.81658935546875 }, { "bits_per_byte": 1.6981439679032087, "learning_rate": 0.00018556729832995555, "loss_nats": 1.1770637035369873, "step": 3830, "unclipped_grad_norm": 0.8478888273239136 }, { "bits_per_byte": 1.649875145129402, "learning_rate": 0.00018501559244109864, "loss_nats": 1.1436063051223755, "step": 3840, "unclipped_grad_norm": 0.8229350447654724 }, { "bits_per_byte": 1.621423539249215, "learning_rate": 0.00018446354454099178, "loss_nats": 1.123885154724121, "step": 3850, "unclipped_grad_norm": 0.8153557181358337 }, { "bits_per_byte": 1.6024271955481633, "learning_rate": 0.00018391116406261048, "loss_nats": 1.1107178926467896, "step": 3860, "unclipped_grad_norm": 0.794911801815033 }, { "bits_per_byte": 1.5664059452729007, "learning_rate": 0.00018335846044461305, "loss_nats": 1.085749864578247, "step": 3870, "unclipped_grad_norm": 0.75998854637146 }, { "bits_per_byte": 1.6739675067581694, "learning_rate": 0.0001828054431311793, "loss_nats": 1.1603058576583862, "step": 3880, "unclipped_grad_norm": 0.8571500182151794 }, { "bits_per_byte": 1.6411153807903556, "learning_rate": 0.00018225212157184934, "loss_nats": 1.137534499168396, "step": 3890, "unclipped_grad_norm": 0.8254691958427429 }, { "bits_per_byte": 1.6427378651185953, "learning_rate": 0.00018169850522136192, "loss_nats": 1.138659119606018, "step": 3900, "unclipped_grad_norm": 0.8312620520591736 }, { "bits_per_byte": 1.6573687731927962, "learning_rate": 0.000181144603539493, "loss_nats": 1.1488004922866821, "step": 3910, "unclipped_grad_norm": 0.8051927089691162 }, { "bits_per_byte": 1.6875147521492837, "learning_rate": 0.00018059042599089412, "loss_nats": 1.1696960926055908, "step": 3920, "unclipped_grad_norm": 0.8500935435295105 }, { "bits_per_byte": 1.685968628118027, "learning_rate": 0.00018003598204493048, "loss_nats": 1.1686244010925293, "step": 3930, "unclipped_grad_norm": 0.8186569213867188 }, { "bits_per_byte": 1.6142946863883005, "learning_rate": 0.00017948128117551936, "loss_nats": 1.1189438104629517, "step": 3940, "unclipped_grad_norm": 0.7923241853713989 }, { "bits_per_byte": 1.7195771337843353, "learning_rate": 0.00017892633286096816, "loss_nats": 1.1919200420379639, "step": 3950, "unclipped_grad_norm": 0.8632136583328247 }, { "bits_per_byte": 1.7081789835984027, "learning_rate": 0.00017837114658381247, "loss_nats": 1.1840194463729858, "step": 3960, "unclipped_grad_norm": 0.8128528594970703 }, { "bits_per_byte": 1.6418629893736563, "learning_rate": 0.00017781573183065386, "loss_nats": 1.1380527019500732, "step": 3970, "unclipped_grad_norm": 0.8211596608161926 }, { "bits_per_byte": 1.6416170541829225, "learning_rate": 0.0001772600980919981, "loss_nats": 1.1378822326660156, "step": 3980, "unclipped_grad_norm": 0.8518010973930359 }, { "bits_per_byte": 1.672920992327669, "learning_rate": 0.0001767042548620927, "loss_nats": 1.1595804691314697, "step": 3990, "unclipped_grad_norm": 0.8055341243743896 }, { "bits_per_byte": 1.6017095119461129, "learning_rate": 0.00017614821163876486, "loss_nats": 1.1102204322814941, "step": 4000, "unclipped_grad_norm": 0.7798590064048767 }, { "bits_per_byte": 1.6685314791296977, "learning_rate": 0.00017559197792325906, "loss_nats": 1.1565378904342651, "step": 4010, "unclipped_grad_norm": 0.8833235502243042 }, { "bits_per_byte": 1.6832612772281446, "learning_rate": 0.0001750355632200748, "loss_nats": 1.166747808456421, "step": 4020, "unclipped_grad_norm": 0.8783014416694641 }, { "bits_per_byte": 1.6355704881438877, "learning_rate": 0.00017447897703680418, "loss_nats": 1.1336910724639893, "step": 4030, "unclipped_grad_norm": 0.831271231174469 }, { "bits_per_byte": 1.6221185211413516, "learning_rate": 0.0001739222288839694, "loss_nats": 1.1243668794631958, "step": 4040, "unclipped_grad_norm": 0.828468382358551 }, { "bits_per_byte": 1.6444031731269, "learning_rate": 0.00017336532827486026, "loss_nats": 1.1398134231567383, "step": 4050, "unclipped_grad_norm": 0.8378818035125732 }, { "bits_per_byte": 1.6213299806871455, "learning_rate": 0.00017280828472537163, "loss_nats": 1.1238203048706055, "step": 4060, "unclipped_grad_norm": 0.808568000793457 }, { "bits_per_byte": 1.749336667724334, "learning_rate": 0.0001722511077538408, "loss_nats": 1.212547779083252, "step": 4070, "unclipped_grad_norm": 0.8772934675216675 }, { "bits_per_byte": 1.5885019322904526, "learning_rate": 0.00017169380688088497, "loss_nats": 1.1010656356811523, "step": 4080, "unclipped_grad_norm": 0.8058938980102539 }, { "bits_per_byte": 1.6013793052564562, "learning_rate": 0.0001711363916292384, "loss_nats": 1.1099915504455566, "step": 4090, "unclipped_grad_norm": 0.8040578365325928 }, { "bits_per_byte": 1.737544161319973, "learning_rate": 0.00017057887152358974, "loss_nats": 1.204373836517334, "step": 4100, "unclipped_grad_norm": 0.8918697834014893 }, { "bits_per_byte": 1.6258001537483717, "learning_rate": 0.0001700212560904194, "loss_nats": 1.1269187927246094, "step": 4110, "unclipped_grad_norm": 0.8254836797714233 }, { "bits_per_byte": 1.6466504704257243, "learning_rate": 0.00016946355485783656, "loss_nats": 1.1413711309432983, "step": 4120, "unclipped_grad_norm": 0.8797640800476074 }, { "bits_per_byte": 1.7012891866221878, "learning_rate": 0.0001689057773554166, "loss_nats": 1.179243803024292, "step": 4130, "unclipped_grad_norm": 0.8626418709754944 }, { "bits_per_byte": 1.6871897049391529, "learning_rate": 0.00016834793311403806, "loss_nats": 1.1694707870483398, "step": 4140, "unclipped_grad_norm": 0.8501971364021301 }, { "bits_per_byte": 1.629858600343432, "learning_rate": 0.00016779003166571981, "loss_nats": 1.1297318935394287, "step": 4150, "unclipped_grad_norm": 0.8614740371704102 }, { "bits_per_byte": 1.6422026551091102, "learning_rate": 0.00016723208254345838, "loss_nats": 1.138288140296936, "step": 4160, "unclipped_grad_norm": 0.8621925115585327 }, { "bits_per_byte": 1.6599294228814925, "learning_rate": 0.00016667409528106481, "loss_nats": 1.1505753993988037, "step": 4170, "unclipped_grad_norm": 0.8789071440696716 }, { "bits_per_byte": 1.6412854716320588, "learning_rate": 0.00016611607941300184, "loss_nats": 1.1376523971557617, "step": 4180, "unclipped_grad_norm": 0.8332215547561646 }, { "bits_per_byte": 1.6753517951149641, "learning_rate": 0.0001655580444742211, "loss_nats": 1.1612653732299805, "step": 4190, "unclipped_grad_norm": 0.8483413457870483 }, { "bits_per_byte": 1.6524258198243482, "learning_rate": 0.00016499999999999997, "loss_nats": 1.1453742980957031, "step": 4200, "unclipped_grad_norm": 0.8514559268951416 }, { "bits_per_byte": 1.6764456047744516, "learning_rate": 0.0001644419555257789, "loss_nats": 1.1620235443115234, "step": 4210, "unclipped_grad_norm": 0.7994921803474426 }, { "bits_per_byte": 1.6553684429806108, "learning_rate": 0.0001638839205869981, "loss_nats": 1.147413969039917, "step": 4220, "unclipped_grad_norm": 0.8137004375457764 }, { "bits_per_byte": 1.654246428166382, "learning_rate": 0.0001633259047189352, "loss_nats": 1.1466362476348877, "step": 4230, "unclipped_grad_norm": 0.8624707460403442 }, { "bits_per_byte": 1.670982919835096, "learning_rate": 0.0001627679174565416, "loss_nats": 1.158237099647522, "step": 4240, "unclipped_grad_norm": 0.83209228515625 }, { "bits_per_byte": 1.605450134602379, "learning_rate": 0.00016220996833428016, "loss_nats": 1.1128132343292236, "step": 4250, "unclipped_grad_norm": 0.8165753483772278 }, { "bits_per_byte": 1.64798711958823, "learning_rate": 0.00016165206688596191, "loss_nats": 1.142297625541687, "step": 4260, "unclipped_grad_norm": 0.8390073180198669 }, { "bits_per_byte": 1.5630770490828, "learning_rate": 0.00016109422264458335, "loss_nats": 1.0834424495697021, "step": 4270, "unclipped_grad_norm": 0.7599806189537048 }, { "bits_per_byte": 1.634252757072977, "learning_rate": 0.00016053644514216336, "loss_nats": 1.1327776908874512, "step": 4280, "unclipped_grad_norm": 0.8349722027778625 }, { "bits_per_byte": 1.622665597953746, "learning_rate": 0.00015997874390958058, "loss_nats": 1.1247460842132568, "step": 4290, "unclipped_grad_norm": 0.8544122576713562 }, { "bits_per_byte": 1.5775992641216556, "learning_rate": 0.00015942112847641018, "loss_nats": 1.0935084819793701, "step": 4300, "unclipped_grad_norm": 0.8515903353691101 }, { "bits_per_byte": 1.5682213941354084, "learning_rate": 0.00015886360837076156, "loss_nats": 1.0870082378387451, "step": 4310, "unclipped_grad_norm": 0.8175749182701111 }, { "bits_per_byte": 1.709660270170284, "learning_rate": 0.00015830619311911495, "loss_nats": 1.1850461959838867, "step": 4320, "unclipped_grad_norm": 0.9185000061988831 }, { "bits_per_byte": 1.6398645509706304, "learning_rate": 0.00015774889224615915, "loss_nats": 1.1366674900054932, "step": 4330, "unclipped_grad_norm": 0.8850803971290588 }, { "bits_per_byte": 1.5869262272432476, "learning_rate": 0.00015719171527462834, "loss_nats": 1.099973440170288, "step": 4340, "unclipped_grad_norm": 0.8476384878158569 }, { "bits_per_byte": 1.5594358323987327, "learning_rate": 0.0001566346717251397, "loss_nats": 1.080918550491333, "step": 4350, "unclipped_grad_norm": 0.8173791170120239 }, { "bits_per_byte": 1.6383724294917448, "learning_rate": 0.00015607777111603056, "loss_nats": 1.1356332302093506, "step": 4360, "unclipped_grad_norm": 0.8429195880889893 }, { "bits_per_byte": 1.6201880158854165, "learning_rate": 0.00015552102296319576, "loss_nats": 1.1230287551879883, "step": 4370, "unclipped_grad_norm": 0.8339970707893372 }, { "bits_per_byte": 1.5560377991829855, "learning_rate": 0.00015496443677992516, "loss_nats": 1.0785632133483887, "step": 4380, "unclipped_grad_norm": 0.7866214513778687 }, { "bits_per_byte": 1.7055524645544258, "learning_rate": 0.0001544080220767409, "loss_nats": 1.1821988821029663, "step": 4390, "unclipped_grad_norm": 0.8178918361663818 }, { "bits_per_byte": 1.6491636529027824, "learning_rate": 0.00015385178836123511, "loss_nats": 1.143113136291504, "step": 4400, "unclipped_grad_norm": 0.8444706797599792 }, { "bits_per_byte": 1.6592194784987309, "learning_rate": 0.00015329574513790726, "loss_nats": 1.150083303451538, "step": 4410, "unclipped_grad_norm": 0.8914594054222107 }, { "bits_per_byte": 1.676016508060549, "learning_rate": 0.00015273990190800186, "loss_nats": 1.1617261171340942, "step": 4420, "unclipped_grad_norm": 0.894481897354126 }, { "bits_per_byte": 1.6150391992838857, "learning_rate": 0.0001521842681693461, "loss_nats": 1.119459867477417, "step": 4430, "unclipped_grad_norm": 0.8697190880775452 }, { "bits_per_byte": 1.63267068866769, "learning_rate": 0.00015162885341618752, "loss_nats": 1.1316810846328735, "step": 4440, "unclipped_grad_norm": 0.8085072636604309 }, { "bits_per_byte": 1.6597337066248108, "learning_rate": 0.00015107366713903178, "loss_nats": 1.150439739227295, "step": 4450, "unclipped_grad_norm": 0.8258711099624634 }, { "bits_per_byte": 1.6374223973283786, "learning_rate": 0.0001505187188244806, "loss_nats": 1.134974718093872, "step": 4460, "unclipped_grad_norm": 0.8935436010360718 }, { "bits_per_byte": 1.6156965169754833, "learning_rate": 0.00014996401795506952, "loss_nats": 1.11991548538208, "step": 4470, "unclipped_grad_norm": 0.8515243530273438 }, { "bits_per_byte": 1.663532975365021, "learning_rate": 0.00014940957400910585, "loss_nats": 1.1530731916427612, "step": 4480, "unclipped_grad_norm": 0.8462327718734741 }, { "bits_per_byte": 1.6896239473794652, "learning_rate": 0.00014885539646050699, "loss_nats": 1.1711580753326416, "step": 4490, "unclipped_grad_norm": 0.8624916672706604 }, { "bits_per_byte": 1.6220770733224936, "learning_rate": 0.00014830149477863805, "loss_nats": 1.124338150024414, "step": 4500, "unclipped_grad_norm": 0.830740213394165 }, { "bits_per_byte": 1.6091536090060587, "learning_rate": 0.00014774787842815066, "loss_nats": 1.1153802871704102, "step": 4510, "unclipped_grad_norm": 0.8425999879837036 }, { "bits_per_byte": 1.6507618876772505, "learning_rate": 0.00014719455686882066, "loss_nats": 1.1442209482192993, "step": 4520, "unclipped_grad_norm": 0.8468135595321655 }, { "bits_per_byte": 1.6238777316770272, "learning_rate": 0.00014664153955538695, "loss_nats": 1.1255862712860107, "step": 4530, "unclipped_grad_norm": 0.8532826900482178 }, { "bits_per_byte": 1.6469891042652731, "learning_rate": 0.00014608883593738946, "loss_nats": 1.1416058540344238, "step": 4540, "unclipped_grad_norm": 0.8736191987991333 }, { "bits_per_byte": 1.6559469926181134, "learning_rate": 0.00014553645545900817, "loss_nats": 1.1478149890899658, "step": 4550, "unclipped_grad_norm": 0.8442186713218689 }, { "bits_per_byte": 1.6446332859137545, "learning_rate": 0.0001449844075589013, "loss_nats": 1.1399729251861572, "step": 4560, "unclipped_grad_norm": 0.9025750756263733 }, { "bits_per_byte": 1.637351540476223, "learning_rate": 0.0001444327016700444, "loss_nats": 1.1349256038665771, "step": 4570, "unclipped_grad_norm": 0.8963979482650757 }, { "bits_per_byte": 1.6074537324849307, "learning_rate": 0.0001438813472195688, "loss_nats": 1.1142020225524902, "step": 4580, "unclipped_grad_norm": 0.8524222373962402 }, { "bits_per_byte": 1.592079171428399, "learning_rate": 0.00014333035362860093, "loss_nats": 1.1035451889038086, "step": 4590, "unclipped_grad_norm": 0.8329559564590454 }, { "bits_per_byte": 1.593515914493707, "learning_rate": 0.00014277973031210088, "loss_nats": 1.1045410633087158, "step": 4600, "unclipped_grad_norm": 0.856568455696106 }, { "bits_per_byte": 1.6099590037600493, "learning_rate": 0.00014222948667870207, "loss_nats": 1.1159385442733765, "step": 4610, "unclipped_grad_norm": 0.8378997445106506 }, { "bits_per_byte": 1.6641679353120065, "learning_rate": 0.00014167963213054991, "loss_nats": 1.1535133123397827, "step": 4620, "unclipped_grad_norm": 0.8407614827156067 }, { "bits_per_byte": 1.64286238055782, "learning_rate": 0.00014113017606314152, "loss_nats": 1.1387454271316528, "step": 4630, "unclipped_grad_norm": 0.9177886247634888 }, { "bits_per_byte": 1.6574955244064822, "learning_rate": 0.00014058112786516518, "loss_nats": 1.148888349533081, "step": 4640, "unclipped_grad_norm": 0.8678322434425354 }, { "bits_per_byte": 1.6864685816840852, "learning_rate": 0.0001400324969183396, "loss_nats": 1.1689709424972534, "step": 4650, "unclipped_grad_norm": 0.8395454287528992 }, { "bits_per_byte": 1.6055636431519484, "learning_rate": 0.00013948429259725406, "loss_nats": 1.1128919124603271, "step": 4660, "unclipped_grad_norm": 0.8130927681922913 }, { "bits_per_byte": 1.6111651180905502, "learning_rate": 0.00013893652426920776, "loss_nats": 1.116774559020996, "step": 4670, "unclipped_grad_norm": 0.8343866467475891 }, { "bits_per_byte": 1.6120890088909852, "learning_rate": 0.00013838920129405016, "loss_nats": 1.117414951324463, "step": 4680, "unclipped_grad_norm": 0.9091700911521912 }, { "bits_per_byte": 1.598018764258597, "learning_rate": 0.0001378423330240207, "loss_nats": 1.1076622009277344, "step": 4690, "unclipped_grad_norm": 0.8087259531021118 }, { "bits_per_byte": 1.6257145063882419, "learning_rate": 0.00013729592880358936, "loss_nats": 1.126859426498413, "step": 4700, "unclipped_grad_norm": 0.8473510146141052 }, { "bits_per_byte": 1.6037934257266175, "learning_rate": 0.00013674999796929659, "loss_nats": 1.111664891242981, "step": 4710, "unclipped_grad_norm": 0.8299785256385803 }, { "bits_per_byte": 1.6360324335441052, "learning_rate": 0.00013620454984959417, "loss_nats": 1.1340112686157227, "step": 4720, "unclipped_grad_norm": 0.8852627873420715 }, { "bits_per_byte": 1.6613310814860245, "learning_rate": 0.00013565959376468542, "loss_nats": 1.1515469551086426, "step": 4730, "unclipped_grad_norm": 0.8318192958831787 }, { "bits_per_byte": 1.6077376758415052, "learning_rate": 0.00013511513902636634, "loss_nats": 1.1143988370895386, "step": 4740, "unclipped_grad_norm": 0.9029622673988342 }, { "bits_per_byte": 1.5803602735986073, "learning_rate": 0.0001345711949378662, "loss_nats": 1.0954222679138184, "step": 4750, "unclipped_grad_norm": 0.8787683248519897 }, { "bits_per_byte": 1.5545300272828713, "learning_rate": 0.00013402777079368853, "loss_nats": 1.077518105506897, "step": 4760, "unclipped_grad_norm": 0.8357442021369934 }, { "bits_per_byte": 1.5563697256991507, "learning_rate": 0.0001334848758794528, "loss_nats": 1.0787932872772217, "step": 4770, "unclipped_grad_norm": 0.8196295499801636 }, { "bits_per_byte": 1.593790398804484, "learning_rate": 0.00013294251947173504, "loss_nats": 1.1047313213348389, "step": 4780, "unclipped_grad_norm": 0.9009830951690674 }, { "bits_per_byte": 1.5906680537780697, "learning_rate": 0.00013240071083790996, "loss_nats": 1.1025670766830444, "step": 4790, "unclipped_grad_norm": 0.8538159728050232 }, { "bits_per_byte": 1.62577985979557, "learning_rate": 0.0001318594592359921, "loss_nats": 1.1269047260284424, "step": 4800, "unclipped_grad_norm": 0.9271508455276489 }, { "bits_per_byte": 1.5952574108163438, "learning_rate": 0.0001313187739144781, "loss_nats": 1.105748176574707, "step": 4810, "unclipped_grad_norm": 0.8505629301071167 }, { "bits_per_byte": 1.5898409892100704, "learning_rate": 0.0001307786641121882, "loss_nats": 1.1019937992095947, "step": 4820, "unclipped_grad_norm": 0.8469997048377991 }, { "bits_per_byte": 1.5923758415011373, "learning_rate": 0.00013023913905810876, "loss_nats": 1.1037508249282837, "step": 4830, "unclipped_grad_norm": 0.8425012826919556 }, { "bits_per_byte": 1.5390662072305417, "learning_rate": 0.00012970020797123432, "loss_nats": 1.0667994022369385, "step": 4840, "unclipped_grad_norm": 0.8439065217971802 }, { "bits_per_byte": 1.6086853002477592, "learning_rate": 0.00012916188006041027, "loss_nats": 1.1150556802749634, "step": 4850, "unclipped_grad_norm": 0.8472049236297607 }, { "bits_per_byte": 1.5891265732783864, "learning_rate": 0.00012862416452417516, "loss_nats": 1.1014986038208008, "step": 4860, "unclipped_grad_norm": 0.8380921483039856 }, { "bits_per_byte": 1.603959560967351, "learning_rate": 0.00012808707055060405, "loss_nats": 1.111780047416687, "step": 4870, "unclipped_grad_norm": 0.8809474110603333 }, { "bits_per_byte": 1.5692639529649388, "learning_rate": 0.000127550607317151, "loss_nats": 1.087730884552002, "step": 4880, "unclipped_grad_norm": 0.7983198761940002 }, { "bits_per_byte": 1.525049621185223, "learning_rate": 0.00012701478399049248, "loss_nats": 1.0570838451385498, "step": 4890, "unclipped_grad_norm": 0.874304473400116 }, { "bits_per_byte": 1.5072996357549742, "learning_rate": 0.0001264796097263708, "loss_nats": 1.0447804927825928, "step": 4900, "unclipped_grad_norm": 0.8955963253974915 }, { "bits_per_byte": 1.5786693401753242, "learning_rate": 0.0001259450936694375, "loss_nats": 1.094250202178955, "step": 4910, "unclipped_grad_norm": 0.8447177410125732 }, { "bits_per_byte": 1.585648052182035, "learning_rate": 0.00012541124495309727, "loss_nats": 1.0990874767303467, "step": 4920, "unclipped_grad_norm": 0.8577683568000793 }, { "bits_per_byte": 1.5737733380705616, "learning_rate": 0.00012487807269935167, "loss_nats": 1.0908565521240234, "step": 4930, "unclipped_grad_norm": 0.8645251393318176 }, { "bits_per_byte": 1.6512401714293001, "learning_rate": 0.00012434558601864349, "loss_nats": 1.14455246925354, "step": 4940, "unclipped_grad_norm": 0.9303934574127197 }, { "bits_per_byte": 1.5141001737353912, "learning_rate": 0.0001238137940097009, "loss_nats": 1.0494942665100098, "step": 4950, "unclipped_grad_norm": 0.8813614249229431 }, { "bits_per_byte": 1.592528046147151, "learning_rate": 0.0001232827057593821, "loss_nats": 1.1038563251495361, "step": 4960, "unclipped_grad_norm": 0.915719747543335 }, { "bits_per_byte": 1.6794828183886887, "learning_rate": 0.00012275233034251992, "loss_nats": 1.1641287803649902, "step": 4970, "unclipped_grad_norm": 0.9402682781219482 }, { "bits_per_byte": 1.543928500735735, "learning_rate": 0.0001222226768217669, "loss_nats": 1.0701696872711182, "step": 4980, "unclipped_grad_norm": 0.9095892310142517 }, { "bits_per_byte": 1.6067152389821258, "learning_rate": 0.00012169375424744034, "loss_nats": 1.1136901378631592, "step": 4990, "unclipped_grad_norm": 0.934682309627533 }, { "bits_per_byte": 1.6283919322968743, "learning_rate": 0.00012116557165736772, "loss_nats": 1.1287152767181396, "step": 5000, "unclipped_grad_norm": 0.8894299268722534 }, { "bits_per_byte": 1.5848010376265358, "learning_rate": 0.00012063813807673214, "loss_nats": 1.098500370979309, "step": 5010, "unclipped_grad_norm": 0.8762696385383606 }, { "bits_per_byte": 1.5972207647585936, "learning_rate": 0.00012011146251791832, "loss_nats": 1.1071090698242188, "step": 5020, "unclipped_grad_norm": 0.87384033203125 }, { "bits_per_byte": 1.5759778116893213, "learning_rate": 0.0001195855539803584, "loss_nats": 1.0923845767974854, "step": 5030, "unclipped_grad_norm": 0.875623881816864 }, { "bits_per_byte": 1.5723771829108575, "learning_rate": 0.00011906042145037817, "loss_nats": 1.0898888111114502, "step": 5040, "unclipped_grad_norm": 0.8759424686431885 }, { "bits_per_byte": 1.6142312247901323, "learning_rate": 0.0001185360739010438, "loss_nats": 1.1188998222351074, "step": 5050, "unclipped_grad_norm": 0.886820912361145 }, { "bits_per_byte": 1.5584469321562717, "learning_rate": 0.00011801252029200808, "loss_nats": 1.080233097076416, "step": 5060, "unclipped_grad_norm": 0.880901038646698 }, { "bits_per_byte": 1.6299523308881525, "learning_rate": 0.00011748976956935777, "loss_nats": 1.1297968626022339, "step": 5070, "unclipped_grad_norm": 0.9343949556350708 }, { "bits_per_byte": 1.651922254622622, "learning_rate": 0.00011696783066546024, "loss_nats": 1.1450252532958984, "step": 5080, "unclipped_grad_norm": 0.9880313277244568 }, { "bits_per_byte": 1.6199948793684975, "learning_rate": 0.00011644671249881147, "loss_nats": 1.1228948831558228, "step": 5090, "unclipped_grad_norm": 1.0230180025100708 }, { "bits_per_byte": 1.5480533326340287, "learning_rate": 0.00011592642397388293, "loss_nats": 1.073028802871704, "step": 5100, "unclipped_grad_norm": 0.9155668020248413 }, { "bits_per_byte": 1.5785750936826513, "learning_rate": 0.00011540697398097015, "loss_nats": 1.0941848754882812, "step": 5110, "unclipped_grad_norm": 0.8749560713768005 }, { "bits_per_byte": 1.553970911684916, "learning_rate": 0.00011488837139604016, "loss_nats": 1.0771305561065674, "step": 5120, "unclipped_grad_norm": 0.9114576578140259 }, { "bits_per_byte": 1.6072428817549729, "learning_rate": 0.00011437062508058039, "loss_nats": 1.114055871963501, "step": 5130, "unclipped_grad_norm": 0.89686119556427 }, { "bits_per_byte": 1.5469013928385495, "learning_rate": 0.00011385374388144673, "loss_nats": 1.072230339050293, "step": 5140, "unclipped_grad_norm": 0.9089201092720032 }, { "bits_per_byte": 1.5509646549478342, "learning_rate": 0.00011333773663071286, "loss_nats": 1.0750467777252197, "step": 5150, "unclipped_grad_norm": 0.9353929162025452 }, { "bits_per_byte": 1.5786652125917033, "learning_rate": 0.00011282261214551897, "loss_nats": 1.0942473411560059, "step": 5160, "unclipped_grad_norm": 0.9100342988967896 }, { "bits_per_byte": 1.5893422395225683, "learning_rate": 0.00011230837922792112, "loss_nats": 1.1016480922698975, "step": 5170, "unclipped_grad_norm": 0.8825995922088623 }, { "bits_per_byte": 1.611850984902191, "learning_rate": 0.0001117950466647412, "loss_nats": 1.1172499656677246, "step": 5180, "unclipped_grad_norm": 0.8858869671821594 }, { "bits_per_byte": 1.5765690880429875, "learning_rate": 0.00011128262322741627, "loss_nats": 1.092794418334961, "step": 5190, "unclipped_grad_norm": 0.893822431564331 }, { "bits_per_byte": 1.5180597303062058, "learning_rate": 0.00011077111767184913, "loss_nats": 1.0522388219833374, "step": 5200, "unclipped_grad_norm": 0.9365838766098022 }, { "bits_per_byte": 1.6136103674205176, "learning_rate": 0.00011026053873825837, "loss_nats": 1.118469476699829, "step": 5210, "unclipped_grad_norm": 0.9187290668487549 }, { "bits_per_byte": 1.578151672396227, "learning_rate": 0.0001097508951510293, "loss_nats": 1.0938913822174072, "step": 5220, "unclipped_grad_norm": 0.9324937462806702 }, { "bits_per_byte": 1.6149896682804372, "learning_rate": 0.0001092421956185646, "loss_nats": 1.1194255352020264, "step": 5230, "unclipped_grad_norm": 0.9387596249580383 }, { "bits_per_byte": 1.5729148006774545, "learning_rate": 0.00010873444883313581, "loss_nats": 1.090261459350586, "step": 5240, "unclipped_grad_norm": 0.8688122630119324 }, { "bits_per_byte": 1.5356076361216906, "learning_rate": 0.00010822766347073447, "loss_nats": 1.0644021034240723, "step": 5250, "unclipped_grad_norm": 0.890831708908081 }, { "bits_per_byte": 1.5141386978491844, "learning_rate": 0.00010772184819092424, "loss_nats": 1.0495209693908691, "step": 5260, "unclipped_grad_norm": 0.8596166968345642 }, { "bits_per_byte": 1.6116535488190005, "learning_rate": 0.0001072170116366926, "loss_nats": 1.1171131134033203, "step": 5270, "unclipped_grad_norm": 0.92216956615448 }, { "bits_per_byte": 1.5994467362087108, "learning_rate": 0.00010671316243430345, "loss_nats": 1.1086519956588745, "step": 5280, "unclipped_grad_norm": 0.9085080623626709 }, { "bits_per_byte": 1.5769201046333985, "learning_rate": 0.00010621030919314944, "loss_nats": 1.093037724494934, "step": 5290, "unclipped_grad_norm": 0.8931360244750977 }, { "bits_per_byte": 1.537008606795619, "learning_rate": 0.000105708460505605, "loss_nats": 1.065373182296753, "step": 5300, "unclipped_grad_norm": 0.8922452330589294 }, { "bits_per_byte": 1.5906943671236518, "learning_rate": 0.00010520762494687964, "loss_nats": 1.1025853157043457, "step": 5310, "unclipped_grad_norm": 0.887484073638916 }, { "bits_per_byte": 1.5623235930893702, "learning_rate": 0.00010470781107487111, "loss_nats": 1.0829201936721802, "step": 5320, "unclipped_grad_norm": 0.9026566743850708 }, { "bits_per_byte": 1.6297041599229574, "learning_rate": 0.00010420902743001955, "loss_nats": 1.129624843597412, "step": 5330, "unclipped_grad_norm": 0.9325461387634277 }, { "bits_per_byte": 1.5387120949524151, "learning_rate": 0.00010371128253516117, "loss_nats": 1.0665539503097534, "step": 5340, "unclipped_grad_norm": 0.8923580050468445 }, { "bits_per_byte": 1.5882112816104945, "learning_rate": 0.000103214584895383, "loss_nats": 1.1008641719818115, "step": 5350, "unclipped_grad_norm": 0.9464812278747559 }, { "bits_per_byte": 1.4963152758270173, "learning_rate": 0.00010271894299787715, "loss_nats": 1.037166714668274, "step": 5360, "unclipped_grad_norm": 0.8723886609077454 }, { "bits_per_byte": 1.5320489711100373, "learning_rate": 0.00010222436531179622, "loss_nats": 1.0619354248046875, "step": 5370, "unclipped_grad_norm": 0.9227972626686096 }, { "bits_per_byte": 1.5933907111238788, "learning_rate": 0.0001017308602881082, "loss_nats": 1.1044542789459229, "step": 5380, "unclipped_grad_norm": 0.9233055710792542 }, { "bits_per_byte": 1.5140970780476757, "learning_rate": 0.00010123843635945242, "loss_nats": 1.0494921207427979, "step": 5390, "unclipped_grad_norm": 0.8707529902458191 }, { "bits_per_byte": 1.5792117734561455, "learning_rate": 0.00010074710193999505, "loss_nats": 1.0946261882781982, "step": 5400, "unclipped_grad_norm": 0.9207195043563843 }, { "bits_per_byte": 1.5488073045754112, "learning_rate": 0.0001002568654252858, "loss_nats": 1.0735514163970947, "step": 5410, "unclipped_grad_norm": 0.8852476477622986 }, { "bits_per_byte": 1.5115954184082252, "learning_rate": 9.976773519211404e-05, "loss_nats": 1.0477581024169922, "step": 5420, "unclipped_grad_norm": 0.9259113073348999 }, { "bits_per_byte": 1.5742989170515984, "learning_rate": 9.92797195983659e-05, "loss_nats": 1.0912208557128906, "step": 5430, "unclipped_grad_norm": 0.9123548865318298 }, { "bits_per_byte": 1.5198012266288425, "learning_rate": 9.879282698288142e-05, "loss_nats": 1.0534459352493286, "step": 5440, "unclipped_grad_norm": 0.8873674273490906 }, { "bits_per_byte": 1.5575636292614403, "learning_rate": 9.830706566531197e-05, "loss_nats": 1.0796208381652832, "step": 5450, "unclipped_grad_norm": 0.851129949092865 }, { "bits_per_byte": 1.5371788696199733, "learning_rate": 9.782244394597826e-05, "loss_nats": 1.0654911994934082, "step": 5460, "unclipped_grad_norm": 0.8905500173568726 }, { "bits_per_byte": 1.5449696837040585, "learning_rate": 9.733897010572829e-05, "loss_nats": 1.0708913803100586, "step": 5470, "unclipped_grad_norm": 0.8742530345916748 }, { "bits_per_byte": 1.5184021477640735, "learning_rate": 9.685665240579612e-05, "loss_nats": 1.052476167678833, "step": 5480, "unclipped_grad_norm": 0.9149489998817444 }, { "bits_per_byte": 1.6054506505503316, "learning_rate": 9.637549908766041e-05, "loss_nats": 1.1128135919570923, "step": 5490, "unclipped_grad_norm": 1.0173310041427612 }, { "bits_per_byte": 1.5652256283400292, "learning_rate": 9.589551837290389e-05, "loss_nats": 1.08493173122406, "step": 5500, "unclipped_grad_norm": 0.9233388304710388 }, { "bits_per_byte": 1.5242242764437322, "learning_rate": 9.541671846307257e-05, "loss_nats": 1.0565117597579956, "step": 5510, "unclipped_grad_norm": 0.8623188734054565 }, { "bits_per_byte": 1.4956853033769069, "learning_rate": 9.493910753953597e-05, "loss_nats": 1.0367300510406494, "step": 5520, "unclipped_grad_norm": 0.8464505672454834 }, { "bits_per_byte": 1.501370017918526, "learning_rate": 9.446269376334689e-05, "loss_nats": 1.040670394897461, "step": 5530, "unclipped_grad_norm": 0.9552764296531677 }, { "bits_per_byte": 1.6303358521995766, "learning_rate": 9.39874852751024e-05, "loss_nats": 1.1300626993179321, "step": 5540, "unclipped_grad_norm": 0.930858314037323 }, { "bits_per_byte": 1.5750513411491232, "learning_rate": 9.351349019480433e-05, "loss_nats": 1.0917423963546753, "step": 5550, "unclipped_grad_norm": 0.9512235522270203 }, { "bits_per_byte": 1.6036597952068972, "learning_rate": 9.304071662172086e-05, "loss_nats": 1.111572265625, "step": 5560, "unclipped_grad_norm": 0.9797965884208679 }, { "bits_per_byte": 1.5364850916063928, "learning_rate": 9.256917263424787e-05, "loss_nats": 1.0650103092193604, "step": 5570, "unclipped_grad_norm": 0.8889045715332031 }, { "bits_per_byte": 1.5818145588943031, "learning_rate": 9.209886628977114e-05, "loss_nats": 1.0964303016662598, "step": 5580, "unclipped_grad_norm": 0.91837477684021 }, { "bits_per_byte": 1.571553042047923, "learning_rate": 9.162980562452843e-05, "loss_nats": 1.0893175601959229, "step": 5590, "unclipped_grad_norm": 0.9484079480171204 }, { "bits_per_byte": 1.5589711352761015, "learning_rate": 9.116199865347238e-05, "loss_nats": 1.0805964469909668, "step": 5600, "unclipped_grad_norm": 0.9200659394264221 }, { "bits_per_byte": 1.4898029807694462, "learning_rate": 9.069545337013333e-05, "loss_nats": 1.032652735710144, "step": 5610, "unclipped_grad_norm": 0.898281455039978 }, { "bits_per_byte": 1.5617056594248202, "learning_rate": 9.0230177746483e-05, "loss_nats": 1.0824918746948242, "step": 5620, "unclipped_grad_norm": 0.9515117406845093 }, { "bits_per_byte": 1.481016903084818, "learning_rate": 8.976617973279797e-05, "loss_nats": 1.0265626907348633, "step": 5630, "unclipped_grad_norm": 0.8537728786468506 }, { "bits_per_byte": 1.471526900410208, "learning_rate": 8.930346725752419e-05, "loss_nats": 1.0199847221374512, "step": 5640, "unclipped_grad_norm": 0.851723313331604 }, { "bits_per_byte": 1.5525594500692852, "learning_rate": 8.88420482271411e-05, "loss_nats": 1.0761522054672241, "step": 5650, "unclipped_grad_norm": 0.9591006636619568 }, { "bits_per_byte": 1.4925528113740916, "learning_rate": 8.838193052602691e-05, "loss_nats": 1.0345587730407715, "step": 5660, "unclipped_grad_norm": 0.9846746325492859 }, { "bits_per_byte": 1.5646704683430441, "learning_rate": 8.792312201632352e-05, "loss_nats": 1.0845469236373901, "step": 5670, "unclipped_grad_norm": 0.9337268471717834 }, { "bits_per_byte": 1.5783896963850212, "learning_rate": 8.746563053780261e-05, "loss_nats": 1.0940563678741455, "step": 5680, "unclipped_grad_norm": 1.0177373886108398 }, { "bits_per_byte": 1.5898387534356093, "learning_rate": 8.700946390773114e-05, "loss_nats": 1.1019922494888306, "step": 5690, "unclipped_grad_norm": 0.9374083876609802 }, { "bits_per_byte": 1.5923674143512452, "learning_rate": 8.655462992073833e-05, "loss_nats": 1.1037449836730957, "step": 5700, "unclipped_grad_norm": 0.9426813721656799 }, { "bits_per_byte": 1.5582474322812707, "learning_rate": 8.610113634868199e-05, "loss_nats": 1.080094814300537, "step": 5710, "unclipped_grad_norm": 0.968785285949707 }, { "bits_per_byte": 1.5819518010496918, "learning_rate": 8.564899094051613e-05, "loss_nats": 1.0965254306793213, "step": 5720, "unclipped_grad_norm": 0.9166642427444458 }, { "bits_per_byte": 1.5197234904706525, "learning_rate": 8.519820142215816e-05, "loss_nats": 1.0533920526504517, "step": 5730, "unclipped_grad_norm": 0.9203759431838989 }, { "bits_per_byte": 1.6015581672133536, "learning_rate": 8.474877549635728e-05, "loss_nats": 1.1101155281066895, "step": 5740, "unclipped_grad_norm": 0.934572160243988 }, { "bits_per_byte": 1.4943809869527633, "learning_rate": 8.430072084256247e-05, "loss_nats": 1.0358259677886963, "step": 5750, "unclipped_grad_norm": 0.8915490508079529 }, { "bits_per_byte": 1.50218005620409, "learning_rate": 8.385404511679161e-05, "loss_nats": 1.0412318706512451, "step": 5760, "unclipped_grad_norm": 0.9438080787658691 }, { "bits_per_byte": 1.5178872317073904, "learning_rate": 8.340875595150034e-05, "loss_nats": 1.052119255065918, "step": 5770, "unclipped_grad_norm": 0.9708113670349121 }, { "bits_per_byte": 1.6077987296825615, "learning_rate": 8.296486095545203e-05, "loss_nats": 1.114441156387329, "step": 5780, "unclipped_grad_norm": 1.0216169357299805 }, { "bits_per_byte": 1.454466909375221, "learning_rate": 8.252236771358729e-05, "loss_nats": 1.0081596374511719, "step": 5790, "unclipped_grad_norm": 0.8829967975616455 }, { "bits_per_byte": 1.5674770532224742, "learning_rate": 8.208128378689484e-05, "loss_nats": 1.0864923000335693, "step": 5800, "unclipped_grad_norm": 0.9498817920684814 }, { "bits_per_byte": 1.4688064788388602, "learning_rate": 8.16416167122819e-05, "loss_nats": 1.018099069595337, "step": 5810, "unclipped_grad_norm": 0.9071933031082153 }, { "bits_per_byte": 1.4812057400354652, "learning_rate": 8.120337400244578e-05, "loss_nats": 1.02669358253479, "step": 5820, "unclipped_grad_norm": 0.9959194660186768 }, { "bits_per_byte": 1.5459386339590195, "learning_rate": 8.076656314574521e-05, "loss_nats": 1.0715630054473877, "step": 5830, "unclipped_grad_norm": 0.9805806875228882 }, { "bits_per_byte": 1.5386066695874363, "learning_rate": 8.03311916060725e-05, "loss_nats": 1.0664808750152588, "step": 5840, "unclipped_grad_norm": 0.9813235998153687 }, { "bits_per_byte": 1.4904921152514534, "learning_rate": 7.989726682272606e-05, "loss_nats": 1.033130407333374, "step": 5850, "unclipped_grad_norm": 0.8884730339050293 }, { "bits_per_byte": 1.505957139182339, "learning_rate": 7.946479621028318e-05, "loss_nats": 1.0438499450683594, "step": 5860, "unclipped_grad_norm": 0.9569032192230225 }, { "bits_per_byte": 1.5707541826346652, "learning_rate": 7.903378715847343e-05, "loss_nats": 1.0887638330459595, "step": 5870, "unclipped_grad_norm": 0.9548988342285156 }, { "bits_per_byte": 1.5056532458382643, "learning_rate": 7.860424703205224e-05, "loss_nats": 1.0436393022537231, "step": 5880, "unclipped_grad_norm": 0.9100088477134705 }, { "bits_per_byte": 1.5621944341185723, "learning_rate": 7.817618317067534e-05, "loss_nats": 1.0828306674957275, "step": 5890, "unclipped_grad_norm": 1.0087047815322876 }, { "bits_per_byte": 1.564086587243365, "learning_rate": 7.774960288877295e-05, "loss_nats": 1.0841422080993652, "step": 5900, "unclipped_grad_norm": 0.9735300540924072 }, { "bits_per_byte": 1.5205851235514751, "learning_rate": 7.732451347542522e-05, "loss_nats": 1.053989291191101, "step": 5910, "unclipped_grad_norm": 0.928986668586731 }, { "bits_per_byte": 1.4827504882055151, "learning_rate": 7.690092219423722e-05, "loss_nats": 1.0277643203735352, "step": 5920, "unclipped_grad_norm": 0.9661484956741333 }, { "bits_per_byte": 1.5936927126587939, "learning_rate": 7.647883628321536e-05, "loss_nats": 1.104663610458374, "step": 5930, "unclipped_grad_norm": 0.9932063817977905 }, { "bits_per_byte": 1.5284586612906255, "learning_rate": 7.605826295464316e-05, "loss_nats": 1.0594468116760254, "step": 5940, "unclipped_grad_norm": 1.0194250345230103 }, { "bits_per_byte": 1.5388254315193337, "learning_rate": 7.563920939495851e-05, "loss_nats": 1.0666325092315674, "step": 5950, "unclipped_grad_norm": 0.9446147084236145 }, { "bits_per_byte": 1.4754302186541906, "learning_rate": 7.522168276463047e-05, "loss_nats": 1.0226902961730957, "step": 5960, "unclipped_grad_norm": 0.9241304993629456 }, { "bits_per_byte": 1.5277755462013984, "learning_rate": 7.480569019803719e-05, "loss_nats": 1.0589733123779297, "step": 5970, "unclipped_grad_norm": 0.9580891728401184 }, { "bits_per_byte": 1.5012076662961116, "learning_rate": 7.439123880334388e-05, "loss_nats": 1.040557861328125, "step": 5980, "unclipped_grad_norm": 0.954113245010376 }, { "bits_per_byte": 1.4896813890352862, "learning_rate": 7.39783356623813e-05, "loss_nats": 1.0325684547424316, "step": 5990, "unclipped_grad_norm": 0.8945097923278809 }, { "bits_per_byte": 1.5812009247960246, "learning_rate": 7.356698783052497e-05, "loss_nats": 1.0960049629211426, "step": 6000, "unclipped_grad_norm": 1.0032626390457153 }, { "bits_per_byte": 1.556720054358958, "learning_rate": 7.315720233657431e-05, "loss_nats": 1.0790361166000366, "step": 6010, "unclipped_grad_norm": 0.9715147614479065 }, { "bits_per_byte": 1.5090167105411882, "learning_rate": 7.274898618263286e-05, "loss_nats": 1.0459706783294678, "step": 6020, "unclipped_grad_norm": 0.9192096590995789 }, { "bits_per_byte": 1.4969094758857484, "learning_rate": 7.234234634398831e-05, "loss_nats": 1.0375785827636719, "step": 6030, "unclipped_grad_norm": 0.9740291833877563 }, { "bits_per_byte": 1.5263979651679873, "learning_rate": 7.193728976899362e-05, "loss_nats": 1.058018445968628, "step": 6040, "unclipped_grad_norm": 0.9185873866081238 }, { "bits_per_byte": 1.5301623214300724, "learning_rate": 7.153382337894797e-05, "loss_nats": 1.0606276988983154, "step": 6050, "unclipped_grad_norm": 0.9192250370979309 }, { "bits_per_byte": 1.5237477125181913, "learning_rate": 7.113195406797887e-05, "loss_nats": 1.0561814308166504, "step": 6060, "unclipped_grad_norm": 0.9670439958572388 }, { "bits_per_byte": 1.4068365721927425, "learning_rate": 7.073168870292397e-05, "loss_nats": 0.9751448035240173, "step": 6070, "unclipped_grad_norm": 0.917002260684967 }, { "bits_per_byte": 1.4967151354902735, "learning_rate": 7.033303412321404e-05, "loss_nats": 1.0374438762664795, "step": 6080, "unclipped_grad_norm": 0.9365084171295166 }, { "bits_per_byte": 1.4969400887976019, "learning_rate": 6.993599714075586e-05, "loss_nats": 1.037599802017212, "step": 6090, "unclipped_grad_norm": 0.9915803074836731 }, { "bits_per_byte": 1.5907201645212812, "learning_rate": 6.954058453981609e-05, "loss_nats": 1.1026031970977783, "step": 6100, "unclipped_grad_norm": 1.0093804597854614 }, { "bits_per_byte": 1.5313995646203795, "learning_rate": 6.914680307690504e-05, "loss_nats": 1.0614852905273438, "step": 6110, "unclipped_grad_norm": 1.0098049640655518 }, { "bits_per_byte": 1.4730523865233613, "learning_rate": 6.875465948066139e-05, "loss_nats": 1.0210421085357666, "step": 6120, "unclipped_grad_norm": 0.9365134239196777 }, { "bits_per_byte": 1.5512489422697104, "learning_rate": 6.836416045173733e-05, "loss_nats": 1.0752438306808472, "step": 6130, "unclipped_grad_norm": 0.9237281680107117 }, { "bits_per_byte": 1.497698360305256, "learning_rate": 6.797531266268377e-05, "loss_nats": 1.0381253957748413, "step": 6140, "unclipped_grad_norm": 0.9138990044593811 }, { "bits_per_byte": 1.4838755987074594, "learning_rate": 6.758812275783663e-05, "loss_nats": 1.0285441875457764, "step": 6150, "unclipped_grad_norm": 0.9429982900619507 }, { "bits_per_byte": 1.551023816979731, "learning_rate": 6.720259735320298e-05, "loss_nats": 1.0750877857208252, "step": 6160, "unclipped_grad_norm": 1.0046567916870117 }, { "bits_per_byte": 1.4928049379402566, "learning_rate": 6.681874303634842e-05, "loss_nats": 1.034733533859253, "step": 6170, "unclipped_grad_norm": 0.964278519153595 }, { "bits_per_byte": 1.4789379768011883, "learning_rate": 6.643656636628402e-05, "loss_nats": 1.0251216888427734, "step": 6180, "unclipped_grad_norm": 0.9606515169143677 }, { "bits_per_byte": 1.501445346319604, "learning_rate": 6.605607387335466e-05, "loss_nats": 1.0407226085662842, "step": 6190, "unclipped_grad_norm": 1.0252128839492798 }, { "bits_per_byte": 1.4915814533620186, "learning_rate": 6.567727205912724e-05, "loss_nats": 1.0338854789733887, "step": 6200, "unclipped_grad_norm": 0.8806238770484924 }, { "bits_per_byte": 1.5186611536362729, "learning_rate": 6.530016739627959e-05, "loss_nats": 1.0526556968688965, "step": 6210, "unclipped_grad_norm": 0.989686906337738 }, { "bits_per_byte": 1.539449556559315, "learning_rate": 6.492476632848993e-05, "loss_nats": 1.0670651197433472, "step": 6220, "unclipped_grad_norm": 0.9960981011390686 }, { "bits_per_byte": 1.4808485320696232, "learning_rate": 6.455107527032678e-05, "loss_nats": 1.026445984840393, "step": 6230, "unclipped_grad_norm": 0.9705929160118103 }, { "bits_per_byte": 1.552468127281677, "learning_rate": 6.417910060713926e-05, "loss_nats": 1.0760889053344727, "step": 6240, "unclipped_grad_norm": 1.0799938440322876 }, { "bits_per_byte": 1.4757860507588256, "learning_rate": 6.380884869494799e-05, "loss_nats": 1.0229369401931763, "step": 6250, "unclipped_grad_norm": 0.9792200922966003 }, { "bits_per_byte": 1.445374530589406, "learning_rate": 6.34403258603367e-05, "loss_nats": 1.0018572807312012, "step": 6260, "unclipped_grad_norm": 0.9422674179077148 }, { "bits_per_byte": 1.5013631386124917, "learning_rate": 6.30735384003437e-05, "loss_nats": 1.040665626525879, "step": 6270, "unclipped_grad_norm": 0.9675420522689819 }, { "bits_per_byte": 1.5119997496204038, "learning_rate": 6.270849258235484e-05, "loss_nats": 1.048038363456726, "step": 6280, "unclipped_grad_norm": 0.9240654110908508 }, { "bits_per_byte": 1.5072342823476461, "learning_rate": 6.234519464399581e-05, "loss_nats": 1.0447351932525635, "step": 6290, "unclipped_grad_norm": 0.9724128842353821 }, { "bits_per_byte": 1.4760084243263911, "learning_rate": 6.198365079302614e-05, "loss_nats": 1.0230910778045654, "step": 6300, "unclipped_grad_norm": 0.9724760055541992 }, { "bits_per_byte": 1.539534687971492, "learning_rate": 6.16238672072326e-05, "loss_nats": 1.0671241283416748, "step": 6310, "unclipped_grad_norm": 1.0172570943832397 }, { "bits_per_byte": 1.498671266161187, "learning_rate": 6.126585003432406e-05, "loss_nats": 1.03879976272583, "step": 6320, "unclipped_grad_norm": 0.9810369610786438 }, { "bits_per_byte": 1.5103010769778318, "learning_rate": 6.090960539182614e-05, "loss_nats": 1.046860933303833, "step": 6330, "unclipped_grad_norm": 0.9735086560249329 }, { "bits_per_byte": 1.4197871237766881, "learning_rate": 6.0555139366976935e-05, "loss_nats": 0.9841214418411255, "step": 6340, "unclipped_grad_norm": 0.9049187302589417 }, { "bits_per_byte": 1.5345103868091858, "learning_rate": 6.020245801662274e-05, "loss_nats": 1.0636415481567383, "step": 6350, "unclipped_grad_norm": 0.9579975008964539 }, { "bits_per_byte": 1.5509206273892133, "learning_rate": 5.985156736711483e-05, "loss_nats": 1.0750162601470947, "step": 6360, "unclipped_grad_norm": 0.989890456199646 }, { "bits_per_byte": 1.4729708667468522, "learning_rate": 5.950247341420618e-05, "loss_nats": 1.0209856033325195, "step": 6370, "unclipped_grad_norm": 1.021422266960144 }, { "bits_per_byte": 1.4625091620948676, "learning_rate": 5.915518212294935e-05, "loss_nats": 1.0137341022491455, "step": 6380, "unclipped_grad_norm": 1.0111291408538818 }, { "bits_per_byte": 1.576993369242666, "learning_rate": 5.880969942759425e-05, "loss_nats": 1.0930885076522827, "step": 6390, "unclipped_grad_norm": 1.0084502696990967 }, { "bits_per_byte": 1.5426105976821733, "learning_rate": 5.846603123148688e-05, "loss_nats": 1.0692561864852905, "step": 6400, "unclipped_grad_norm": 1.060890793800354 }, { "bits_per_byte": 1.542634847235945, "learning_rate": 5.812418340696858e-05, "loss_nats": 1.0692729949951172, "step": 6410, "unclipped_grad_norm": 1.0786292552947998 }, { "bits_per_byte": 1.4677484415707522, "learning_rate": 5.778416179527541e-05, "loss_nats": 1.0173656940460205, "step": 6420, "unclipped_grad_norm": 1.0185354948043823 }, { "bits_per_byte": 1.5249625979638863, "learning_rate": 5.7445972206438635e-05, "loss_nats": 1.057023525238037, "step": 6430, "unclipped_grad_norm": 0.9899823069572449 }, { "bits_per_byte": 1.447091949340922, "learning_rate": 5.710962041918516e-05, "loss_nats": 1.0030477046966553, "step": 6440, "unclipped_grad_norm": 0.9853183031082153 }, { "bits_per_byte": 1.4600725118974434, "learning_rate": 5.677511218083908e-05, "loss_nats": 1.01204514503479, "step": 6450, "unclipped_grad_norm": 0.9329874515533447 }, { "bits_per_byte": 1.4881399085356029, "learning_rate": 5.644245320722316e-05, "loss_nats": 1.031499981880188, "step": 6460, "unclipped_grad_norm": 0.9553210735321045 }, { "bits_per_byte": 1.5093300629310604, "learning_rate": 5.61116491825615e-05, "loss_nats": 1.0461878776550293, "step": 6470, "unclipped_grad_norm": 1.0880736112594604 }, { "bits_per_byte": 1.4620568477230984, "learning_rate": 5.5782705759382104e-05, "loss_nats": 1.013420581817627, "step": 6480, "unclipped_grad_norm": 0.978254497051239 }, { "bits_per_byte": 1.5301041912940807, "learning_rate": 5.5455628558420546e-05, "loss_nats": 1.0605874061584473, "step": 6490, "unclipped_grad_norm": 1.0214275121688843 }, { "bits_per_byte": 1.5703515712489953, "learning_rate": 5.513042316852368e-05, "loss_nats": 1.088484764099121, "step": 6500, "unclipped_grad_norm": 1.024044156074524 }, { "bits_per_byte": 1.5418132861127734, "learning_rate": 5.480709514655443e-05, "loss_nats": 1.068703532218933, "step": 6510, "unclipped_grad_norm": 1.0472792387008667 }, { "bits_per_byte": 1.5326505664227554, "learning_rate": 5.448565001729653e-05, "loss_nats": 1.0623524188995361, "step": 6520, "unclipped_grad_norm": 0.9873340129852295 }, { "bits_per_byte": 1.493115194642413, "learning_rate": 5.41660932733604e-05, "loss_nats": 1.0349485874176025, "step": 6530, "unclipped_grad_norm": 0.9528921246528625 }, { "bits_per_byte": 1.4717057623671055, "learning_rate": 5.384843037508902e-05, "loss_nats": 1.020108699798584, "step": 6540, "unclipped_grad_norm": 0.9596993327140808 }, { "bits_per_byte": 1.532975785615537, "learning_rate": 5.353266675046498e-05, "loss_nats": 1.0625778436660767, "step": 6550, "unclipped_grad_norm": 0.9932572245597839 }, { "bits_per_byte": 1.5382425823155597, "learning_rate": 5.321880779501729e-05, "loss_nats": 1.0662285089492798, "step": 6560, "unclipped_grad_norm": 1.0132641792297363 }, { "bits_per_byte": 1.5160394501065202, "learning_rate": 5.290685887172962e-05, "loss_nats": 1.0508384704589844, "step": 6570, "unclipped_grad_norm": 0.9644629955291748 }, { "bits_per_byte": 1.503854823258192, "learning_rate": 5.2596825310948295e-05, "loss_nats": 1.0423927307128906, "step": 6580, "unclipped_grad_norm": 0.9727261662483215 }, { "bits_per_byte": 1.5231672710715294, "learning_rate": 5.228871241029154e-05, "loss_nats": 1.0557790994644165, "step": 6590, "unclipped_grad_norm": 1.029261589050293 }, { "bits_per_byte": 1.415425385776831, "learning_rate": 5.198252543455865e-05, "loss_nats": 0.9810981154441833, "step": 6600, "unclipped_grad_norm": 0.9430753588676453 }, { "bits_per_byte": 1.4673490978554489, "learning_rate": 5.167826961564027e-05, "loss_nats": 1.0170888900756836, "step": 6610, "unclipped_grad_norm": 0.9699459671974182 }, { "bits_per_byte": 1.5770974187464382, "learning_rate": 5.137595015242884e-05, "loss_nats": 1.093160629272461, "step": 6620, "unclipped_grad_norm": 1.053364872932434 }, { "bits_per_byte": 1.520643081704816, "learning_rate": 5.1075572210729925e-05, "loss_nats": 1.0540294647216797, "step": 6630, "unclipped_grad_norm": 1.0116336345672607 }, { "bits_per_byte": 1.4843385760035823, "learning_rate": 5.07771409231737e-05, "loss_nats": 1.028865098953247, "step": 6640, "unclipped_grad_norm": 0.9483272433280945 }, { "bits_per_byte": 1.481005552229861, "learning_rate": 5.048066138912756e-05, "loss_nats": 1.026554822921753, "step": 6650, "unclipped_grad_norm": 0.9847526550292969 }, { "bits_per_byte": 1.5181830418668742, "learning_rate": 5.018613867460863e-05, "loss_nats": 1.0523242950439453, "step": 6660, "unclipped_grad_norm": 1.0222764015197754 }, { "bits_per_byte": 1.483456133022005, "learning_rate": 4.989357781219755e-05, "loss_nats": 1.028253436088562, "step": 6670, "unclipped_grad_norm": 0.9609436392784119 }, { "bits_per_byte": 1.5070292790278177, "learning_rate": 4.960298380095222e-05, "loss_nats": 1.044593095779419, "step": 6680, "unclipped_grad_norm": 1.0481340885162354 }, { "bits_per_byte": 1.4999456576040804, "learning_rate": 4.931436160632256e-05, "loss_nats": 1.0396831035614014, "step": 6690, "unclipped_grad_norm": 1.0560411214828491 }, { "bits_per_byte": 1.5028304945896531, "learning_rate": 4.9027716160065485e-05, "loss_nats": 1.0416827201843262, "step": 6700, "unclipped_grad_norm": 1.0250314474105835 }, { "bits_per_byte": 1.5313854620430087, "learning_rate": 4.874305236016084e-05, "loss_nats": 1.0614755153656006, "step": 6710, "unclipped_grad_norm": 0.9873788356781006 }, { "bits_per_byte": 1.5848861690387128, "learning_rate": 4.846037507072752e-05, "loss_nats": 1.0985593795776367, "step": 6720, "unclipped_grad_norm": 1.065596103668213 }, { "bits_per_byte": 1.4535899698384716, "learning_rate": 4.817968912194049e-05, "loss_nats": 1.0075517892837524, "step": 6730, "unclipped_grad_norm": 0.9767746329307556 }, { "bits_per_byte": 1.4828168735087481, "learning_rate": 4.790099930994813e-05, "loss_nats": 1.0278103351593018, "step": 6740, "unclipped_grad_norm": 0.9589064717292786 }, { "bits_per_byte": 1.4599156637198565, "learning_rate": 4.762431039679047e-05, "loss_nats": 1.0119364261627197, "step": 6750, "unclipped_grad_norm": 0.9927419424057007 }, { "bits_per_byte": 1.4484303183299363, "learning_rate": 4.734962711031755e-05, "loss_nats": 1.0039753913879395, "step": 6760, "unclipped_grad_norm": 0.9825301766395569 }, { "bits_per_byte": 1.469467236183475, "learning_rate": 4.707695414410892e-05, "loss_nats": 1.018557071685791, "step": 6770, "unclipped_grad_norm": 1.0425026416778564 }, { "bits_per_byte": 1.4605650702095145, "learning_rate": 4.680629615739322e-05, "loss_nats": 1.0123865604400635, "step": 6780, "unclipped_grad_norm": 1.0209197998046875 }, { "bits_per_byte": 1.494792713418929, "learning_rate": 4.653765777496863e-05, "loss_nats": 1.0361113548278809, "step": 6790, "unclipped_grad_norm": 0.9975507855415344 }, { "bits_per_byte": 1.4806505800384802, "learning_rate": 4.6271043587123986e-05, "loss_nats": 1.0263087749481201, "step": 6800, "unclipped_grad_norm": 1.0428087711334229 }, { "bits_per_byte": 1.4777557680591573, "learning_rate": 4.600645814956006e-05, "loss_nats": 1.0243022441864014, "step": 6810, "unclipped_grad_norm": 0.9905012249946594 }, { "bits_per_byte": 1.4914748241184836, "learning_rate": 4.574390598331204e-05, "loss_nats": 1.0338115692138672, "step": 6820, "unclipped_grad_norm": 1.0114696025848389 }, { "bits_per_byte": 1.4751165222990168, "learning_rate": 4.548339157467199e-05, "loss_nats": 1.022472858428955, "step": 6830, "unclipped_grad_norm": 0.9909161329269409 }, { "bits_per_byte": 1.462379831141419, "learning_rate": 4.52249193751124e-05, "loss_nats": 1.0136444568634033, "step": 6840, "unclipped_grad_norm": 1.0066962242126465 }, { "bits_per_byte": 1.542950607382929, "learning_rate": 4.4968493801209985e-05, "loss_nats": 1.0694918632507324, "step": 6850, "unclipped_grad_norm": 1.0904512405395508 }, { "bits_per_byte": 1.5094851912821385, "learning_rate": 4.471411923457034e-05, "loss_nats": 1.046295404434204, "step": 6860, "unclipped_grad_norm": 1.0577117204666138 }, { "bits_per_byte": 1.4431473552607326, "learning_rate": 4.446180002175291e-05, "loss_nats": 1.0003135204315186, "step": 6870, "unclipped_grad_norm": 0.931652843952179 }, { "bits_per_byte": 1.4721159409894133, "learning_rate": 4.421154047419693e-05, "loss_nats": 1.0203930139541626, "step": 6880, "unclipped_grad_norm": 1.0439471006393433 }, { "bits_per_byte": 1.4601951355275085, "learning_rate": 4.396334486814753e-05, "loss_nats": 1.0121301412582397, "step": 6890, "unclipped_grad_norm": 0.9945755004882812 }, { "bits_per_byte": 1.5078214311176918, "learning_rate": 4.371721744458285e-05, "loss_nats": 1.0451421737670898, "step": 6900, "unclipped_grad_norm": 1.0577389001846313 }, { "bits_per_byte": 1.51248938422741, "learning_rate": 4.347316240914145e-05, "loss_nats": 1.0483777523040771, "step": 6910, "unclipped_grad_norm": 1.042457938194275 }, { "bits_per_byte": 1.4546720846777004, "learning_rate": 4.3231183932050546e-05, "loss_nats": 1.008301854133606, "step": 6920, "unclipped_grad_norm": 1.0035841464996338 }, { "bits_per_byte": 1.4388924184870613, "learning_rate": 4.299128614805464e-05, "loss_nats": 0.9973642230033875, "step": 6930, "unclipped_grad_norm": 1.0526496171951294 }, { "bits_per_byte": 1.5114688391771902, "learning_rate": 4.2753473156344965e-05, "loss_nats": 1.0476703643798828, "step": 6940, "unclipped_grad_norm": 1.0117003917694092 }, { "bits_per_byte": 1.4256293744264978, "learning_rate": 4.251774902048942e-05, "loss_nats": 0.9881709814071655, "step": 6950, "unclipped_grad_norm": 1.020249843597412 }, { "bits_per_byte": 1.4392803253560824, "learning_rate": 4.228411776836306e-05, "loss_nats": 0.9976330995559692, "step": 6960, "unclipped_grad_norm": 1.007938265800476 }, { "bits_per_byte": 1.425250496646647, "learning_rate": 4.2052583392079427e-05, "loss_nats": 0.9879083633422852, "step": 6970, "unclipped_grad_norm": 1.0789695978164673 }, { "bits_per_byte": 1.5310960152416067, "learning_rate": 4.182314984792214e-05, "loss_nats": 1.0612748861312866, "step": 6980, "unclipped_grad_norm": 1.0816211700439453 }, { "bits_per_byte": 1.4834984407541174, "learning_rate": 4.159582105627748e-05, "loss_nats": 1.0282827615737915, "step": 6990, "unclipped_grad_norm": 1.0542293787002563 }, { "bits_per_byte": 1.494992041311279, "learning_rate": 4.137060090156724e-05, "loss_nats": 1.0362495183944702, "step": 7000, "unclipped_grad_norm": 1.040958046913147 }, { "bits_per_byte": 1.4039343649594325, "learning_rate": 4.114749323218249e-05, "loss_nats": 0.9731331467628479, "step": 7010, "unclipped_grad_norm": 1.0020650625228882 }, { "bits_per_byte": 1.4906596263533938, "learning_rate": 4.0926501860417706e-05, "loss_nats": 1.0332465171813965, "step": 7020, "unclipped_grad_norm": 1.0739529132843018 }, { "bits_per_byte": 1.4642052549976767, "learning_rate": 4.070763056240571e-05, "loss_nats": 1.0149097442626953, "step": 7030, "unclipped_grad_norm": 1.0350828170776367 }, { "bits_per_byte": 1.4574465088014192, "learning_rate": 4.049088307805306e-05, "loss_nats": 1.0102249383926392, "step": 7040, "unclipped_grad_norm": 1.0089728832244873 }, { "bits_per_byte": 1.4534154074478458, "learning_rate": 4.027626311097629e-05, "loss_nats": 1.0074307918548584, "step": 7050, "unclipped_grad_norm": 1.0608088970184326 }, { "bits_per_byte": 1.4416395833606184, "learning_rate": 4.006377432843846e-05, "loss_nats": 0.9992684125900269, "step": 7060, "unclipped_grad_norm": 0.9864199757575989 }, { "bits_per_byte": 1.4906718371216052, "learning_rate": 3.985342036128656e-05, "loss_nats": 1.0332549810409546, "step": 7070, "unclipped_grad_norm": 1.0150113105773926 }, { "bits_per_byte": 1.4931912109740944, "learning_rate": 3.964520480388956e-05, "loss_nats": 1.035001277923584, "step": 7080, "unclipped_grad_norm": 1.0090219974517822 }, { "bits_per_byte": 1.4509221749582877, "learning_rate": 3.943913121407678e-05, "loss_nats": 1.0057026147842407, "step": 7090, "unclipped_grad_norm": 1.0420514345169067 }, { "bits_per_byte": 1.4389093587781714, "learning_rate": 3.923520311307735e-05, "loss_nats": 0.9973759651184082, "step": 7100, "unclipped_grad_norm": 1.0253499746322632 }, { "bits_per_byte": 1.41304419998431, "learning_rate": 3.9033423985459816e-05, "loss_nats": 0.979447603225708, "step": 7110, "unclipped_grad_norm": 0.9803867936134338 }, { "bits_per_byte": 1.4722433801337027, "learning_rate": 3.88337972790728e-05, "loss_nats": 1.0204813480377197, "step": 7120, "unclipped_grad_norm": 1.0725315809249878 }, { "bits_per_byte": 1.5197811046586915, "learning_rate": 3.8636326404985845e-05, "loss_nats": 1.0534319877624512, "step": 7130, "unclipped_grad_norm": 1.0701161623001099 }, { "bits_per_byte": 1.465326925846604, "learning_rate": 3.844101473743143e-05, "loss_nats": 1.0156872272491455, "step": 7140, "unclipped_grad_norm": 1.0326207876205444 }, { "bits_per_byte": 1.5100518741167317, "learning_rate": 3.824786561374701e-05, "loss_nats": 1.046688199043274, "step": 7150, "unclipped_grad_norm": 1.0673091411590576 }, { "bits_per_byte": 1.5022746466620645, "learning_rate": 3.805688233431824e-05, "loss_nats": 1.041297435760498, "step": 7160, "unclipped_grad_norm": 1.059691071510315 }, { "bits_per_byte": 1.514605802728928, "learning_rate": 3.786806816252236e-05, "loss_nats": 1.049844741821289, "step": 7170, "unclipped_grad_norm": 1.085069179534912 }, { "bits_per_byte": 1.4489440305080636, "learning_rate": 3.768142632467269e-05, "loss_nats": 1.0043314695358276, "step": 7180, "unclipped_grad_norm": 0.9880333542823792 }, { "bits_per_byte": 1.5073161460894569, "learning_rate": 3.7496960009963236e-05, "loss_nats": 1.0447919368743896, "step": 7190, "unclipped_grad_norm": 1.0678606033325195 }, { "bits_per_byte": 1.4434393818018978, "learning_rate": 3.7314672370414325e-05, "loss_nats": 1.0005159378051758, "step": 7200, "unclipped_grad_norm": 1.0179466009140015 }, { "bits_per_byte": 1.4303239848470999, "learning_rate": 3.713456652081882e-05, "loss_nats": 0.9914250373840332, "step": 7210, "unclipped_grad_norm": 1.005952000617981 }, { "bits_per_byte": 1.4493132772594661, "learning_rate": 3.695664553868873e-05, "loss_nats": 1.0045874118804932, "step": 7220, "unclipped_grad_norm": 1.0151865482330322 }, { "bits_per_byte": 1.475039818036732, "learning_rate": 3.678091246420277e-05, "loss_nats": 1.0224196910858154, "step": 7230, "unclipped_grad_norm": 1.0841153860092163 }, { "bits_per_byte": 1.5119313005253603, "learning_rate": 3.660737030015427e-05, "loss_nats": 1.0479909181594849, "step": 7240, "unclipped_grad_norm": 1.09213387966156 }, { "bits_per_byte": 1.494972951237033, "learning_rate": 3.6436022011900056e-05, "loss_nats": 1.03623628616333, "step": 7250, "unclipped_grad_norm": 1.0454415082931519 }, { "bits_per_byte": 1.4613747645297765, "learning_rate": 3.626687052730954e-05, "loss_nats": 1.0129477977752686, "step": 7260, "unclipped_grad_norm": 1.0101300477981567 }, { "bits_per_byte": 1.4686286487778681, "learning_rate": 3.6099918736714955e-05, "loss_nats": 1.0179758071899414, "step": 7270, "unclipped_grad_norm": 1.0528877973556519 }, { "bits_per_byte": 1.4749450555961066, "learning_rate": 3.5935169492861716e-05, "loss_nats": 1.022354006767273, "step": 7280, "unclipped_grad_norm": 1.0265198945999146 }, { "bits_per_byte": 1.473522931056122, "learning_rate": 3.5772625610859886e-05, "loss_nats": 1.0213682651519775, "step": 7290, "unclipped_grad_norm": 1.066664457321167 }, { "bits_per_byte": 1.4559291058728565, "learning_rate": 3.5612289868135887e-05, "loss_nats": 1.0091731548309326, "step": 7300, "unclipped_grad_norm": 1.0080223083496094 }, { "bits_per_byte": 1.445694074354709, "learning_rate": 3.545416500438522e-05, "loss_nats": 1.0020787715911865, "step": 7310, "unclipped_grad_norm": 1.0805282592773438 }, { "bits_per_byte": 1.4250067972303744, "learning_rate": 3.52982537215255e-05, "loss_nats": 0.9877394437789917, "step": 7320, "unclipped_grad_norm": 1.0912657976150513 }, { "bits_per_byte": 1.4783119599520478, "learning_rate": 3.5144558683650396e-05, "loss_nats": 1.0246877670288086, "step": 7330, "unclipped_grad_norm": 1.0849530696868896 }, { "bits_per_byte": 1.4380544330207323, "learning_rate": 3.499308251698404e-05, "loss_nats": 0.9967833757400513, "step": 7340, "unclipped_grad_norm": 1.042580485343933 }, { "bits_per_byte": 1.4567071553853599, "learning_rate": 3.484382780983616e-05, "loss_nats": 1.0097124576568604, "step": 7350, "unclipped_grad_norm": 1.0958631038665771 }, { "bits_per_byte": 1.4481006275882324, "learning_rate": 3.469679711255795e-05, "loss_nats": 1.0037468671798706, "step": 7360, "unclipped_grad_norm": 1.0337251424789429 }, { "bits_per_byte": 1.4027950658887918, "learning_rate": 3.4551992937498284e-05, "loss_nats": 0.9723434448242188, "step": 7370, "unclipped_grad_norm": 1.0347329378128052 }, { "bits_per_byte": 1.4160821015291507, "learning_rate": 3.440941775896104e-05, "loss_nats": 0.981553316116333, "step": 7380, "unclipped_grad_norm": 1.0372161865234375 }, { "bits_per_byte": 1.421894943145663, "learning_rate": 3.4269074013162596e-05, "loss_nats": 0.9855824708938599, "step": 7390, "unclipped_grad_norm": 1.038377046585083 }, { "bits_per_byte": 1.4461825050831596, "learning_rate": 3.41309640981904e-05, "loss_nats": 1.0024173259735107, "step": 7400, "unclipped_grad_norm": 1.057859182357788 }, { "bits_per_byte": 1.475963364871865, "learning_rate": 3.3995090373961795e-05, "loss_nats": 1.0230598449707031, "step": 7410, "unclipped_grad_norm": 1.0708297491073608 }, { "bits_per_byte": 1.469500256852441, "learning_rate": 3.3861455162183876e-05, "loss_nats": 1.0185799598693848, "step": 7420, "unclipped_grad_norm": 1.062455654144287 }, { "bits_per_byte": 1.41574011402791, "learning_rate": 3.3730060746313664e-05, "loss_nats": 0.9813162684440613, "step": 7430, "unclipped_grad_norm": 1.0484251976013184 }, { "bits_per_byte": 1.3821044351027654, "learning_rate": 3.3600909371519225e-05, "loss_nats": 0.9580017924308777, "step": 7440, "unclipped_grad_norm": 1.030645728111267 }, { "bits_per_byte": 1.4987479704234719, "learning_rate": 3.3474003244641194e-05, "loss_nats": 1.0388529300689697, "step": 7450, "unclipped_grad_norm": 1.0465614795684814 }, { "bits_per_byte": 1.4339140366925356, "learning_rate": 3.334934453415514e-05, "loss_nats": 0.993913471698761, "step": 7460, "unclipped_grad_norm": 1.0371458530426025 }, { "bits_per_byte": 1.4012647642614147, "learning_rate": 3.3226935370134486e-05, "loss_nats": 0.9712827205657959, "step": 7470, "unclipped_grad_norm": 1.0733681917190552 }, { "bits_per_byte": 1.5010874504231586, "learning_rate": 3.31067778442141e-05, "loss_nats": 1.040474534034729, "step": 7480, "unclipped_grad_norm": 1.0303142070770264 }, { "bits_per_byte": 1.4650142613873354, "learning_rate": 3.298887400955457e-05, "loss_nats": 1.0154705047607422, "step": 7490, "unclipped_grad_norm": 1.0710126161575317 }, { "bits_per_byte": 1.472811954777455, "learning_rate": 3.287322588080714e-05, "loss_nats": 1.0208754539489746, "step": 7500, "unclipped_grad_norm": 1.0901119709014893 }, { "bits_per_byte": 1.4645168875610401, "learning_rate": 3.275983543407924e-05, "loss_nats": 1.0151257514953613, "step": 7510, "unclipped_grad_norm": 1.024397611618042 }, { "bits_per_byte": 1.4743771688829574, "learning_rate": 3.2648704606900735e-05, "loss_nats": 1.0219603776931763, "step": 7520, "unclipped_grad_norm": 1.0934079885482788 }, { "bits_per_byte": 1.46691879726299, "learning_rate": 3.253983529819086e-05, "loss_nats": 1.0167906284332275, "step": 7530, "unclipped_grad_norm": 1.081713318824768 }, { "bits_per_byte": 1.4640198577000465, "learning_rate": 3.243322936822571e-05, "loss_nats": 1.0147812366485596, "step": 7540, "unclipped_grad_norm": 1.0780924558639526 }, { "bits_per_byte": 1.4520784143200383, "learning_rate": 3.232888863860648e-05, "loss_nats": 1.0065040588378906, "step": 7550, "unclipped_grad_norm": 1.04312264919281 }, { "bits_per_byte": 1.3812490793886991, "learning_rate": 3.222681489222838e-05, "loss_nats": 0.9574089050292969, "step": 7560, "unclipped_grad_norm": 1.0638409852981567 }, { "bits_per_byte": 1.4099118799641457, "learning_rate": 3.212700987325009e-05, "loss_nats": 0.9772764444351196, "step": 7570, "unclipped_grad_norm": 1.0123975276947021 }, { "bits_per_byte": 1.3981073347742243, "learning_rate": 3.202947528706403e-05, "loss_nats": 0.9690941572189331, "step": 7580, "unclipped_grad_norm": 1.0492113828659058 }, { "bits_per_byte": 1.472455950690169, "learning_rate": 3.1934212800267116e-05, "loss_nats": 1.0206286907196045, "step": 7590, "unclipped_grad_norm": 1.0436540842056274 }, { "bits_per_byte": 1.5055485084038889, "learning_rate": 3.1841224040632484e-05, "loss_nats": 1.0435667037963867, "step": 7600, "unclipped_grad_norm": 1.140743374824524 }, { "bits_per_byte": 1.4701950667619266, "learning_rate": 3.175051059708143e-05, "loss_nats": 1.01906156539917, "step": 7610, "unclipped_grad_norm": 1.082565426826477 }, { "bits_per_byte": 1.458967179400348, "learning_rate": 3.1662074019656414e-05, "loss_nats": 1.0112789869308472, "step": 7620, "unclipped_grad_norm": 1.0694279670715332 }, { "bits_per_byte": 1.4490995028244436, "learning_rate": 3.157591581949455e-05, "loss_nats": 1.0044392347335815, "step": 7630, "unclipped_grad_norm": 1.0964275598526 }, { "bits_per_byte": 1.4314776444690875, "learning_rate": 3.149203746880175e-05, "loss_nats": 0.9922246932983398, "step": 7640, "unclipped_grad_norm": 1.15219247341156 }, { "bits_per_byte": 1.4066524647649938, "learning_rate": 3.141044040082763e-05, "loss_nats": 0.9750171899795532, "step": 7650, "unclipped_grad_norm": 1.064915418624878 }, { "bits_per_byte": 1.416584462852321, "learning_rate": 3.133112600984091e-05, "loss_nats": 0.9819015264511108, "step": 7660, "unclipped_grad_norm": 1.141569972038269 }, { "bits_per_byte": 1.4608237321164121, "learning_rate": 3.125409565110571e-05, "loss_nats": 1.0125658512115479, "step": 7670, "unclipped_grad_norm": 1.0876517295837402 }, { "bits_per_byte": 1.4539619683122877, "learning_rate": 3.117935064085835e-05, "loss_nats": 1.0078096389770508, "step": 7680, "unclipped_grad_norm": 1.0938481092453003 }, { "bits_per_byte": 1.433180616677931, "learning_rate": 3.110689225628479e-05, "loss_nats": 0.9934051036834717, "step": 7690, "unclipped_grad_norm": 1.070209264755249 }, { "bits_per_byte": 1.4952543148538446, "learning_rate": 3.1036721735498926e-05, "loss_nats": 1.0364313125610352, "step": 7700, "unclipped_grad_norm": 1.0864635705947876 }, { "bits_per_byte": 1.412102165014209, "learning_rate": 3.096884027752132e-05, "loss_nats": 0.9787946343421936, "step": 7710, "unclipped_grad_norm": 1.040504813194275 }, { "bits_per_byte": 1.4180753804526507, "learning_rate": 3.090324904225885e-05, "loss_nats": 0.9829349517822266, "step": 7720, "unclipped_grad_norm": 1.0190634727478027 }, { "bits_per_byte": 1.4255394275000965, "learning_rate": 3.083994915048472e-05, "loss_nats": 0.9881086349487305, "step": 7730, "unclipped_grad_norm": 1.0293750762939453 }, { "bits_per_byte": 1.468768986620972, "learning_rate": 3.077894168381942e-05, "loss_nats": 1.0180730819702148, "step": 7740, "unclipped_grad_norm": 1.0531129837036133 }, { "bits_per_byte": 1.4037740771288283, "learning_rate": 3.0720227684712254e-05, "loss_nats": 0.9730220437049866, "step": 7750, "unclipped_grad_norm": 1.1033527851104736 }, { "bits_per_byte": 1.4111542826339787, "learning_rate": 3.0663808156423456e-05, "loss_nats": 0.9781376123428345, "step": 7760, "unclipped_grad_norm": 1.0707266330718994 }, { "bits_per_byte": 1.4583370349675868, "learning_rate": 3.060968406300711e-05, "loss_nats": 1.010842204093933, "step": 7770, "unclipped_grad_norm": 1.0769577026367188 }, { "bits_per_byte": 1.4516912813729461, "learning_rate": 3.055785632929463e-05, "loss_nats": 1.0062357187271118, "step": 7780, "unclipped_grad_norm": 1.1653907299041748 }, { "bits_per_byte": 1.4586383485718983, "learning_rate": 3.0508325840878984e-05, "loss_nats": 1.011051058769226, "step": 7790, "unclipped_grad_norm": 1.0739812850952148 }, { "bits_per_byte": 1.4426457678594913, "learning_rate": 3.046109344409957e-05, "loss_nats": 0.9999658465385437, "step": 7800, "unclipped_grad_norm": 1.0972295999526978 }, { "bits_per_byte": 1.4198621082124645, "learning_rate": 3.0416159946027727e-05, "loss_nats": 0.9841734170913696, "step": 7810, "unclipped_grad_norm": 1.075659990310669 }, { "bits_per_byte": 1.4512271001982673, "learning_rate": 3.0373526114452946e-05, "loss_nats": 1.0059139728546143, "step": 7820, "unclipped_grad_norm": 1.1029958724975586 }, { "bits_per_byte": 1.4583721194283628, "learning_rate": 3.0333192677869803e-05, "loss_nats": 1.0108665227890015, "step": 7830, "unclipped_grad_norm": 1.081362009048462 }, { "bits_per_byte": 1.4242661539444337, "learning_rate": 3.029516032546544e-05, "loss_nats": 0.9872260689735413, "step": 7840, "unclipped_grad_norm": 1.0629743337631226 }, { "bits_per_byte": 1.423362471105475, "learning_rate": 3.02594297071078e-05, "loss_nats": 0.9865996837615967, "step": 7850, "unclipped_grad_norm": 1.0624163150787354 }, { "bits_per_byte": 1.4807702799634808, "learning_rate": 3.0226001433334603e-05, "loss_nats": 1.0263917446136475, "step": 7860, "unclipped_grad_norm": 1.0853880643844604 }, { "bits_per_byte": 1.4488509878939468, "learning_rate": 3.019487607534278e-05, "loss_nats": 1.0042669773101807, "step": 7870, "unclipped_grad_norm": 1.1049087047576904 }, { "bits_per_byte": 1.4220910033676464, "learning_rate": 3.016605416497886e-05, "loss_nats": 0.9857183694839478, "step": 7880, "unclipped_grad_norm": 1.0985115766525269 }, { "bits_per_byte": 1.4594506226319235, "learning_rate": 3.0139536194729734e-05, "loss_nats": 1.0116140842437744, "step": 7890, "unclipped_grad_norm": 1.0736068487167358 }, { "bits_per_byte": 1.4105420243969071, "learning_rate": 3.0115322617714362e-05, "loss_nats": 0.9777132272720337, "step": 7900, "unclipped_grad_norm": 1.1535842418670654 }, { "bits_per_byte": 1.463574250651661, "learning_rate": 3.0093413847675965e-05, "loss_nats": 1.0144723653793335, "step": 7910, "unclipped_grad_norm": 1.1884382963180542 }, { "bits_per_byte": 1.4830432026772835, "learning_rate": 3.0073810258974985e-05, "loss_nats": 1.0279672145843506, "step": 7920, "unclipped_grad_norm": 1.1013721227645874 }, { "bits_per_byte": 1.435883066062264, "learning_rate": 3.0056512186582658e-05, "loss_nats": 0.9952782988548279, "step": 7930, "unclipped_grad_norm": 1.0824542045593262 }, { "bits_per_byte": 1.4594231054077853, "learning_rate": 3.0041519926075316e-05, "loss_nats": 1.0115950107574463, "step": 7940, "unclipped_grad_norm": 1.1213319301605225 }, { "bits_per_byte": 1.408353201199376, "learning_rate": 3.0028833733629313e-05, "loss_nats": 0.9761960506439209, "step": 7950, "unclipped_grad_norm": 1.0871061086654663 }, { "bits_per_byte": 1.4735662706841393, "learning_rate": 3.0018453826016686e-05, "loss_nats": 1.0213983058929443, "step": 7960, "unclipped_grad_norm": 1.1091856956481934 }, { "bits_per_byte": 1.4267374586460069, "learning_rate": 3.001038038060143e-05, "loss_nats": 0.9889390468597412, "step": 7970, "unclipped_grad_norm": 1.0478838682174683 }, { "bits_per_byte": 1.472857186214632, "learning_rate": 3.0004613535336433e-05, "loss_nats": 1.0209068059921265, "step": 7980, "unclipped_grad_norm": 1.2172881364822388 }, { "bits_per_byte": 1.4136101948882995, "learning_rate": 3.000115338876117e-05, "loss_nats": 0.9798399209976196, "step": 7990, "unclipped_grad_norm": 1.1755717992782593 }, { "bits_per_byte": 1.46345644253582, "learning_rate": 3e-05, "loss_nats": 1.0143907070159912, "step": 8000, "unclipped_grad_norm": 1.0856302976608276 } ] }